mirror of
https://github.com/dutchcoders/transfer.sh.git
synced 2024-11-24 05:00:18 +01:00
377 lines
9.7 KiB
Go
377 lines
9.7 KiB
Go
// Copyright 2018 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package httpreplay_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"log"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"cloud.google.com/go/httpreplay"
|
|
"cloud.google.com/go/internal/testutil"
|
|
"cloud.google.com/go/storage"
|
|
"google.golang.org/api/option"
|
|
)
|
|
|
|
func TestIntegration_RecordAndReplay(t *testing.T) {
|
|
httpreplay.DebugHeaders()
|
|
if testing.Short() {
|
|
t.Skip("Integration tests skipped in short mode")
|
|
}
|
|
replayFilename := tempFilename(t, "RecordAndReplay*.replay")
|
|
defer os.Remove(replayFilename)
|
|
projectID := testutil.ProjID()
|
|
if projectID == "" {
|
|
t.Skip("Need project ID. See CONTRIBUTING.md for details.")
|
|
}
|
|
ctx := context.Background()
|
|
|
|
// Record.
|
|
initial := time.Now()
|
|
ibytes, err := json.Marshal(initial)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rec, err := httpreplay.NewRecorder(replayFilename, ibytes)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
hc, err := rec.Client(ctx, option.WithTokenSource(
|
|
testutil.TokenSource(ctx, storage.ScopeFullControl)))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wanta, wantc := run(t, hc)
|
|
testReadCRC(t, hc, "recording")
|
|
if err := rec.Close(); err != nil {
|
|
t.Fatalf("rec.Close: %v", err)
|
|
}
|
|
|
|
// Replay.
|
|
rep, err := httpreplay.NewReplayer(replayFilename)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer rep.Close()
|
|
hc, err = rep.Client(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gota, gotc := run(t, hc)
|
|
testReadCRC(t, hc, "replaying")
|
|
|
|
if diff := testutil.Diff(gota, wanta); diff != "" {
|
|
t.Error(diff)
|
|
}
|
|
if !bytes.Equal(gotc, wantc) {
|
|
t.Errorf("got %q, want %q", gotc, wantc)
|
|
}
|
|
var gotInitial time.Time
|
|
if err := json.Unmarshal(rep.Initial(), &gotInitial); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !gotInitial.Equal(initial) {
|
|
t.Errorf("initial: got %v, want %v", gotInitial, initial)
|
|
}
|
|
}
|
|
|
|
// TODO(jba): test errors
|
|
|
|
func run(t *testing.T, hc *http.Client) (*storage.BucketAttrs, []byte) {
|
|
ctx := context.Background()
|
|
client, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer client.Close()
|
|
b := client.Bucket(testutil.ProjID())
|
|
attrs, err := b.Attrs(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
obj := b.Object("replay-test")
|
|
w := obj.NewWriter(ctx)
|
|
data := []byte{150, 151, 152}
|
|
if _, err := w.Write(data); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := w.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
r, err := obj.NewReader(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer r.Close()
|
|
contents, err := ioutil.ReadAll(r)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
return attrs, contents
|
|
}
|
|
|
|
func testReadCRC(t *testing.T, hc *http.Client, mode string) {
|
|
const (
|
|
// This is an uncompressed file.
|
|
// See https://cloud.google.com/storage/docs/public-datasets/landsat
|
|
uncompressedBucket = "gcp-public-data-landsat"
|
|
uncompressedObject = "LC08/PRE/044/034/LC80440342016259LGN00/LC80440342016259LGN00_MTL.txt"
|
|
|
|
gzippedBucket = "storage-library-test-bucket"
|
|
gzippedObject = "gzipped-text.txt"
|
|
)
|
|
ctx := context.Background()
|
|
client, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
|
|
if err != nil {
|
|
t.Fatalf("%s: %v", mode, err)
|
|
}
|
|
defer client.Close()
|
|
|
|
uncompressedObj := client.Bucket(uncompressedBucket).Object(uncompressedObject)
|
|
gzippedObj := client.Bucket(gzippedBucket).Object(gzippedObject)
|
|
|
|
for _, test := range []struct {
|
|
desc string
|
|
obj *storage.ObjectHandle
|
|
offset, length int64
|
|
readCompressed bool // don't decompress a gzipped file
|
|
|
|
wantErr bool
|
|
wantLen int // length of contents
|
|
}{
|
|
{
|
|
desc: "uncompressed, entire file",
|
|
obj: uncompressedObj,
|
|
offset: 0,
|
|
length: -1,
|
|
readCompressed: false,
|
|
wantLen: 7903,
|
|
},
|
|
{
|
|
desc: "uncompressed, entire file, don't decompress",
|
|
obj: uncompressedObj,
|
|
offset: 0,
|
|
length: -1,
|
|
readCompressed: true,
|
|
wantLen: 7903,
|
|
},
|
|
{
|
|
desc: "uncompressed, suffix",
|
|
obj: uncompressedObj,
|
|
offset: 3,
|
|
length: -1,
|
|
readCompressed: false,
|
|
wantLen: 7900,
|
|
},
|
|
{
|
|
desc: "uncompressed, prefix",
|
|
obj: uncompressedObj,
|
|
offset: 0,
|
|
length: 18,
|
|
readCompressed: false,
|
|
wantLen: 18,
|
|
},
|
|
{
|
|
// When a gzipped file is unzipped by GCS, we can't verify the checksum
|
|
// because it was computed against the zipped contents. There is no
|
|
// header that indicates that a gzipped file is being served unzipped.
|
|
// But our CRC check only happens if there is a Content-Length header,
|
|
// and that header is absent for this read.
|
|
desc: "compressed, entire file, server unzips",
|
|
obj: gzippedObj,
|
|
offset: 0,
|
|
length: -1,
|
|
readCompressed: false,
|
|
wantLen: 11,
|
|
},
|
|
{
|
|
// When we read a gzipped file uncompressed, it's like reading a regular file:
|
|
// the served content and the CRC match.
|
|
desc: "compressed, entire file, read compressed",
|
|
obj: gzippedObj,
|
|
offset: 0,
|
|
length: -1,
|
|
readCompressed: true,
|
|
wantLen: 31,
|
|
},
|
|
{
|
|
desc: "compressed, partial, read compressed",
|
|
obj: gzippedObj,
|
|
offset: 1,
|
|
length: 8,
|
|
readCompressed: true,
|
|
wantLen: 8,
|
|
},
|
|
{
|
|
desc: "uncompressed, HEAD",
|
|
obj: uncompressedObj,
|
|
offset: 0,
|
|
length: 0,
|
|
wantLen: 0,
|
|
},
|
|
{
|
|
desc: "compressed, HEAD",
|
|
obj: gzippedObj,
|
|
offset: 0,
|
|
length: 0,
|
|
wantLen: 0,
|
|
},
|
|
} {
|
|
obj := test.obj.ReadCompressed(test.readCompressed)
|
|
r, err := obj.NewRangeReader(ctx, test.offset, test.length)
|
|
if err != nil {
|
|
if test.wantErr {
|
|
continue
|
|
}
|
|
t.Errorf("%s: %s: %v", mode, test.desc, err)
|
|
continue
|
|
}
|
|
data, err := ioutil.ReadAll(r)
|
|
_ = r.Close()
|
|
if err != nil {
|
|
t.Errorf("%s: %s: %v", mode, test.desc, err)
|
|
continue
|
|
}
|
|
if got, want := len(data), test.wantLen; got != want {
|
|
t.Errorf("%s: %s: len: got %d, want %d", mode, test.desc, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRemoveAndClear(t *testing.T) {
|
|
// Disable logging for this test, since it generates a lot.
|
|
log.SetOutput(ioutil.Discard)
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
fmt.Fprintln(w, "LGTM")
|
|
}))
|
|
defer srv.Close()
|
|
|
|
replayFilename := tempFilename(t, "TestRemoveAndClear*.replay")
|
|
defer os.Remove(replayFilename)
|
|
|
|
ctx := context.Background()
|
|
// Record
|
|
rec, err := httpreplay.NewRecorder(replayFilename, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rec.ClearHeaders("Clear")
|
|
rec.RemoveRequestHeaders("Rem*")
|
|
rec.ClearQueryParams("c")
|
|
rec.RemoveQueryParams("r")
|
|
hc, err := rec.Client(ctx, option.WithoutAuthentication())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
query := "k=1&r=2&c=3"
|
|
req, err := http.NewRequest("GET", srv.URL+"?"+query, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
headers := map[string]string{"Keep": "ok", "Clear": "secret", "Remove": "bye"}
|
|
for k, v := range headers {
|
|
req.Header.Set(k, v)
|
|
}
|
|
if _, err := hc.Do(req); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := rec.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Replay
|
|
// For both headers and query param:
|
|
// - k or Keep must be present and identical
|
|
// - c or Clear must be present, but can be different
|
|
// - r or Remove can be anything
|
|
for _, test := range []struct {
|
|
query string
|
|
headers map[string]string
|
|
wantSuccess bool
|
|
}{
|
|
{query, headers, true}, // same query string and headers
|
|
{query,
|
|
map[string]string{"Keep": "oops", "Clear": "secret", "Remove": "bye"},
|
|
false, // different Keep
|
|
},
|
|
{query, map[string]string{}, false}, // missing Keep and Clear
|
|
{query, map[string]string{"Keep": "ok"}, false}, // missing Clear
|
|
{query, map[string]string{"Keep": "ok", "Clear": "secret"}, true}, // missing Remove is OK
|
|
{
|
|
query,
|
|
map[string]string{"Keep": "ok", "Clear": "secret", "Remove": "whatev"},
|
|
true,
|
|
}, // different Remove is OK
|
|
{query, map[string]string{"Keep": "ok", "Clear": "diff"}, true}, // different Clear is OK
|
|
{"", headers, false}, // no query string
|
|
{"k=x&r=2&c=3", headers, false}, // different k
|
|
{"r=2", headers, false}, // missing k and c
|
|
{"k=1&r=2", headers, false}, // missing c
|
|
{"k=1&c=3", headers, true}, // missing r is OK
|
|
{"k=1&r=x&c=3", headers, true}, // different r is OK,
|
|
{"k=1&r=2&c=x", headers, true}, // different clear is OK
|
|
} {
|
|
rep, err := httpreplay.NewReplayer(replayFilename)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
hc, err = rep.Client(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
url := srv.URL
|
|
if test.query != "" {
|
|
url += "?" + test.query
|
|
}
|
|
req, err = http.NewRequest("GET", url, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for k, v := range test.headers {
|
|
req.Header.Set(k, v)
|
|
}
|
|
resp, err := hc.Do(req)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rep.Close()
|
|
if (resp.StatusCode == 200) != test.wantSuccess {
|
|
t.Errorf("%q, %v: got %d, wanted success=%t",
|
|
test.query, test.headers, resp.StatusCode, test.wantSuccess)
|
|
}
|
|
}
|
|
}
|
|
|
|
func tempFilename(t *testing.T, pattern string) string {
|
|
f, err := ioutil.TempFile("", pattern)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
filename := f.Name()
|
|
if err := f.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return filename
|
|
}
|