mirror of
https://github.com/dutchcoders/transfer.sh.git
synced 2024-11-23 20:50:18 +01:00
372 lines
10 KiB
Go
372 lines
10 KiB
Go
// Copyright 2017 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package bigquery
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"cloud.google.com/go/internal/testutil"
|
|
bq "google.golang.org/api/bigquery/v2"
|
|
)
|
|
|
|
func TestBQToTableMetadata(t *testing.T) {
|
|
aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
|
|
aTimeMillis := aTime.UnixNano() / 1e6
|
|
for _, test := range []struct {
|
|
in *bq.Table
|
|
want *TableMetadata
|
|
}{
|
|
{&bq.Table{}, &TableMetadata{}}, // test minimal case
|
|
{
|
|
&bq.Table{
|
|
CreationTime: aTimeMillis,
|
|
Description: "desc",
|
|
Etag: "etag",
|
|
ExpirationTime: aTimeMillis,
|
|
FriendlyName: "fname",
|
|
Id: "id",
|
|
LastModifiedTime: uint64(aTimeMillis),
|
|
Location: "loc",
|
|
NumBytes: 123,
|
|
NumLongTermBytes: 23,
|
|
NumRows: 7,
|
|
StreamingBuffer: &bq.Streamingbuffer{
|
|
EstimatedBytes: 11,
|
|
EstimatedRows: 3,
|
|
OldestEntryTime: uint64(aTimeMillis),
|
|
},
|
|
TimePartitioning: &bq.TimePartitioning{
|
|
ExpirationMs: 7890,
|
|
Type: "DAY",
|
|
Field: "pfield",
|
|
},
|
|
Clustering: &bq.Clustering{
|
|
Fields: []string{"cfield1", "cfield2"},
|
|
},
|
|
EncryptionConfiguration: &bq.EncryptionConfiguration{KmsKeyName: "keyName"},
|
|
Type: "EXTERNAL",
|
|
View: &bq.ViewDefinition{Query: "view-query"},
|
|
Labels: map[string]string{"a": "b"},
|
|
ExternalDataConfiguration: &bq.ExternalDataConfiguration{
|
|
SourceFormat: "GOOGLE_SHEETS",
|
|
},
|
|
},
|
|
&TableMetadata{
|
|
Description: "desc",
|
|
Name: "fname",
|
|
ViewQuery: "view-query",
|
|
FullID: "id",
|
|
Type: ExternalTable,
|
|
Labels: map[string]string{"a": "b"},
|
|
ExternalDataConfig: &ExternalDataConfig{SourceFormat: GoogleSheets},
|
|
ExpirationTime: aTime.Truncate(time.Millisecond),
|
|
CreationTime: aTime.Truncate(time.Millisecond),
|
|
LastModifiedTime: aTime.Truncate(time.Millisecond),
|
|
NumBytes: 123,
|
|
NumLongTermBytes: 23,
|
|
NumRows: 7,
|
|
TimePartitioning: &TimePartitioning{
|
|
Expiration: 7890 * time.Millisecond,
|
|
Field: "pfield",
|
|
},
|
|
Clustering: &Clustering{
|
|
Fields: []string{"cfield1", "cfield2"},
|
|
},
|
|
StreamingBuffer: &StreamingBuffer{
|
|
EstimatedBytes: 11,
|
|
EstimatedRows: 3,
|
|
OldestEntryTime: aTime,
|
|
},
|
|
EncryptionConfig: &EncryptionConfig{KMSKeyName: "keyName"},
|
|
ETag: "etag",
|
|
},
|
|
},
|
|
} {
|
|
got, err := bqToTableMetadata(test.in)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if diff := testutil.Diff(got, test.want); diff != "" {
|
|
t.Errorf("%+v:\n, -got, +want:\n%s", test.in, diff)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTableMetadataToBQ(t *testing.T) {
|
|
aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
|
|
aTimeMillis := aTime.UnixNano() / 1e6
|
|
sc := Schema{fieldSchema("desc", "name", "STRING", false, true)}
|
|
|
|
for _, test := range []struct {
|
|
in *TableMetadata
|
|
want *bq.Table
|
|
}{
|
|
{nil, &bq.Table{}},
|
|
{&TableMetadata{}, &bq.Table{}},
|
|
{
|
|
&TableMetadata{
|
|
Name: "n",
|
|
Description: "d",
|
|
Schema: sc,
|
|
ExpirationTime: aTime,
|
|
Labels: map[string]string{"a": "b"},
|
|
ExternalDataConfig: &ExternalDataConfig{SourceFormat: Bigtable},
|
|
EncryptionConfig: &EncryptionConfig{KMSKeyName: "keyName"},
|
|
},
|
|
&bq.Table{
|
|
FriendlyName: "n",
|
|
Description: "d",
|
|
Schema: &bq.TableSchema{
|
|
Fields: []*bq.TableFieldSchema{
|
|
bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"),
|
|
},
|
|
},
|
|
ExpirationTime: aTimeMillis,
|
|
Labels: map[string]string{"a": "b"},
|
|
ExternalDataConfiguration: &bq.ExternalDataConfiguration{SourceFormat: "BIGTABLE"},
|
|
EncryptionConfiguration: &bq.EncryptionConfiguration{KmsKeyName: "keyName"},
|
|
},
|
|
},
|
|
{
|
|
&TableMetadata{ViewQuery: "q"},
|
|
&bq.Table{
|
|
View: &bq.ViewDefinition{
|
|
Query: "q",
|
|
UseLegacySql: false,
|
|
ForceSendFields: []string{"UseLegacySql"},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
&TableMetadata{
|
|
ViewQuery: "q",
|
|
UseLegacySQL: true,
|
|
TimePartitioning: &TimePartitioning{},
|
|
},
|
|
&bq.Table{
|
|
View: &bq.ViewDefinition{
|
|
Query: "q",
|
|
UseLegacySql: true,
|
|
},
|
|
TimePartitioning: &bq.TimePartitioning{
|
|
Type: "DAY",
|
|
ExpirationMs: 0,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
&TableMetadata{
|
|
ViewQuery: "q",
|
|
UseStandardSQL: true,
|
|
TimePartitioning: &TimePartitioning{
|
|
Expiration: time.Second,
|
|
Field: "ofDreams",
|
|
},
|
|
Clustering: &Clustering{
|
|
Fields: []string{"cfield1"},
|
|
},
|
|
},
|
|
&bq.Table{
|
|
View: &bq.ViewDefinition{
|
|
Query: "q",
|
|
UseLegacySql: false,
|
|
ForceSendFields: []string{"UseLegacySql"},
|
|
},
|
|
TimePartitioning: &bq.TimePartitioning{
|
|
Type: "DAY",
|
|
ExpirationMs: 1000,
|
|
Field: "ofDreams",
|
|
},
|
|
Clustering: &bq.Clustering{
|
|
Fields: []string{"cfield1"},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
&TableMetadata{ExpirationTime: NeverExpire},
|
|
&bq.Table{ExpirationTime: 0},
|
|
},
|
|
} {
|
|
got, err := test.in.toBQ()
|
|
if err != nil {
|
|
t.Fatalf("%+v: %v", test.in, err)
|
|
}
|
|
if diff := testutil.Diff(got, test.want); diff != "" {
|
|
t.Errorf("%+v:\n-got, +want:\n%s", test.in, diff)
|
|
}
|
|
}
|
|
|
|
// Errors
|
|
for _, in := range []*TableMetadata{
|
|
{Schema: sc, ViewQuery: "q"}, // can't have both schema and query
|
|
{UseLegacySQL: true}, // UseLegacySQL without query
|
|
{UseStandardSQL: true}, // UseStandardSQL without query
|
|
// read-only fields
|
|
{FullID: "x"},
|
|
{Type: "x"},
|
|
{CreationTime: aTime},
|
|
{LastModifiedTime: aTime},
|
|
{NumBytes: 1},
|
|
{NumLongTermBytes: 1},
|
|
{NumRows: 1},
|
|
{StreamingBuffer: &StreamingBuffer{}},
|
|
{ETag: "x"},
|
|
// expiration time outside allowable range is invalid
|
|
// See https://godoc.org/time#Time.UnixNano
|
|
{ExpirationTime: time.Date(1677, 9, 21, 0, 12, 43, 145224192, time.UTC).Add(-1)},
|
|
{ExpirationTime: time.Date(2262, 04, 11, 23, 47, 16, 854775807, time.UTC).Add(1)},
|
|
} {
|
|
_, err := in.toBQ()
|
|
if err == nil {
|
|
t.Errorf("%+v: got nil, want error", in)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTableMetadataToUpdateToBQ(t *testing.T) {
|
|
aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
|
|
for _, test := range []struct {
|
|
tm TableMetadataToUpdate
|
|
want *bq.Table
|
|
}{
|
|
{
|
|
tm: TableMetadataToUpdate{},
|
|
want: &bq.Table{},
|
|
},
|
|
{
|
|
tm: TableMetadataToUpdate{
|
|
Description: "d",
|
|
Name: "n",
|
|
},
|
|
want: &bq.Table{
|
|
Description: "d",
|
|
FriendlyName: "n",
|
|
ForceSendFields: []string{"Description", "FriendlyName"},
|
|
},
|
|
},
|
|
{
|
|
tm: TableMetadataToUpdate{
|
|
Schema: Schema{fieldSchema("desc", "name", "STRING", false, true)},
|
|
ExpirationTime: aTime,
|
|
},
|
|
want: &bq.Table{
|
|
Schema: &bq.TableSchema{
|
|
Fields: []*bq.TableFieldSchema{
|
|
bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"),
|
|
},
|
|
},
|
|
ExpirationTime: aTime.UnixNano() / 1e6,
|
|
ForceSendFields: []string{"Schema", "ExpirationTime"},
|
|
},
|
|
},
|
|
{
|
|
tm: TableMetadataToUpdate{ViewQuery: "q"},
|
|
want: &bq.Table{
|
|
View: &bq.ViewDefinition{Query: "q", ForceSendFields: []string{"Query"}},
|
|
},
|
|
},
|
|
{
|
|
tm: TableMetadataToUpdate{UseLegacySQL: false},
|
|
want: &bq.Table{
|
|
View: &bq.ViewDefinition{
|
|
UseLegacySql: false,
|
|
ForceSendFields: []string{"UseLegacySql"},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
tm: TableMetadataToUpdate{ViewQuery: "q", UseLegacySQL: true},
|
|
want: &bq.Table{
|
|
View: &bq.ViewDefinition{
|
|
Query: "q",
|
|
UseLegacySql: true,
|
|
ForceSendFields: []string{"Query", "UseLegacySql"},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
tm: func() (tm TableMetadataToUpdate) {
|
|
tm.SetLabel("L", "V")
|
|
tm.DeleteLabel("D")
|
|
return tm
|
|
}(),
|
|
want: &bq.Table{
|
|
Labels: map[string]string{"L": "V"},
|
|
NullFields: []string{"Labels.D"},
|
|
},
|
|
},
|
|
{
|
|
tm: TableMetadataToUpdate{ExpirationTime: NeverExpire},
|
|
want: &bq.Table{
|
|
NullFields: []string{"ExpirationTime"},
|
|
},
|
|
},
|
|
{
|
|
tm: TableMetadataToUpdate{TimePartitioning: &TimePartitioning{Expiration: 0}},
|
|
want: &bq.Table{
|
|
TimePartitioning: &bq.TimePartitioning{
|
|
Type: "DAY",
|
|
ForceSendFields: []string{"RequirePartitionFilter"},
|
|
NullFields: []string{"ExpirationMs"},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
tm: TableMetadataToUpdate{TimePartitioning: &TimePartitioning{Expiration: time.Duration(time.Hour)}},
|
|
want: &bq.Table{
|
|
TimePartitioning: &bq.TimePartitioning{
|
|
ExpirationMs: 3600000,
|
|
Type: "DAY",
|
|
ForceSendFields: []string{"RequirePartitionFilter"},
|
|
},
|
|
},
|
|
},
|
|
} {
|
|
got, _ := test.tm.toBQ()
|
|
if !testutil.Equal(got, test.want) {
|
|
t.Errorf("%+v:\ngot %+v\nwant %+v", test.tm, got, test.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTableMetadataToUpdateToBQErrors(t *testing.T) {
|
|
// See https://godoc.org/time#Time.UnixNano
|
|
start := time.Date(1677, 9, 21, 0, 12, 43, 145224192, time.UTC)
|
|
end := time.Date(2262, 04, 11, 23, 47, 16, 854775807, time.UTC)
|
|
|
|
for _, test := range []struct {
|
|
desc string
|
|
aTime time.Time
|
|
wantErr bool
|
|
}{
|
|
{desc: "ignored zero value", aTime: time.Time{}, wantErr: false},
|
|
{desc: "earliest valid time", aTime: start, wantErr: false},
|
|
{desc: "latested valid time", aTime: end, wantErr: false},
|
|
{desc: "invalid times before 1678", aTime: start.Add(-1), wantErr: true},
|
|
{desc: "invalid times after 2262", aTime: end.Add(1), wantErr: true},
|
|
{desc: "valid times after 1678", aTime: start.Add(1), wantErr: false},
|
|
{desc: "valid times before 2262", aTime: end.Add(-1), wantErr: false},
|
|
} {
|
|
tm := &TableMetadataToUpdate{ExpirationTime: test.aTime}
|
|
_, err := tm.toBQ()
|
|
if test.wantErr && err == nil {
|
|
t.Errorf("[%s] got no error, want error", test.desc)
|
|
}
|
|
if !test.wantErr && err != nil {
|
|
t.Errorf("[%s] got error, want no error", test.desc)
|
|
}
|
|
}
|
|
}
|