mirror of
https://github.com/dutchcoders/transfer.sh.git
synced 2024-11-24 05:00:18 +01:00
578 lines
15 KiB
Go
578 lines
15 KiB
Go
// Copyright 2015 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package bigtable
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"math"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"cloud.google.com/go/internal/testutil"
|
|
"github.com/golang/protobuf/proto"
|
|
"google.golang.org/api/iterator"
|
|
btapb "google.golang.org/genproto/googleapis/bigtable/admin/v2"
|
|
)
|
|
|
|
func TestAdminIntegration(t *testing.T) {
|
|
testEnv, err := NewIntegrationEnv()
|
|
if err != nil {
|
|
t.Fatalf("IntegrationEnv: %v", err)
|
|
}
|
|
defer testEnv.Close()
|
|
|
|
timeout := 2 * time.Second
|
|
if testEnv.Config().UseProd {
|
|
timeout = 5 * time.Minute
|
|
}
|
|
ctx, _ := context.WithTimeout(context.Background(), timeout)
|
|
|
|
adminClient, err := testEnv.NewAdminClient()
|
|
if err != nil {
|
|
t.Fatalf("NewAdminClient: %v", err)
|
|
}
|
|
defer adminClient.Close()
|
|
|
|
iAdminClient, err := testEnv.NewInstanceAdminClient()
|
|
if err != nil {
|
|
t.Fatalf("NewInstanceAdminClient: %v", err)
|
|
}
|
|
if iAdminClient != nil {
|
|
defer iAdminClient.Close()
|
|
|
|
iInfo, err := iAdminClient.InstanceInfo(ctx, adminClient.instance)
|
|
if err != nil {
|
|
t.Errorf("InstanceInfo: %v", err)
|
|
}
|
|
if iInfo.Name != adminClient.instance {
|
|
t.Errorf("InstanceInfo returned name %#v, want %#v", iInfo.Name, adminClient.instance)
|
|
}
|
|
}
|
|
|
|
list := func() []string {
|
|
tbls, err := adminClient.Tables(ctx)
|
|
if err != nil {
|
|
t.Fatalf("Fetching list of tables: %v", err)
|
|
}
|
|
sort.Strings(tbls)
|
|
return tbls
|
|
}
|
|
containsAll := func(got, want []string) bool {
|
|
gotSet := make(map[string]bool)
|
|
|
|
for _, s := range got {
|
|
gotSet[s] = true
|
|
}
|
|
for _, s := range want {
|
|
if !gotSet[s] {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
defer adminClient.DeleteTable(ctx, "mytable")
|
|
|
|
if err := adminClient.CreateTable(ctx, "mytable"); err != nil {
|
|
t.Fatalf("Creating table: %v", err)
|
|
}
|
|
|
|
defer adminClient.DeleteTable(ctx, "myothertable")
|
|
|
|
if err := adminClient.CreateTable(ctx, "myothertable"); err != nil {
|
|
t.Fatalf("Creating table: %v", err)
|
|
}
|
|
|
|
if got, want := list(), []string{"myothertable", "mytable"}; !containsAll(got, want) {
|
|
t.Errorf("adminClient.Tables returned %#v, want %#v", got, want)
|
|
}
|
|
|
|
must(adminClient.WaitForReplication(ctx, "mytable"))
|
|
|
|
if err := adminClient.DeleteTable(ctx, "myothertable"); err != nil {
|
|
t.Fatalf("Deleting table: %v", err)
|
|
}
|
|
tables := list()
|
|
if got, want := tables, []string{"mytable"}; !containsAll(got, want) {
|
|
t.Errorf("adminClient.Tables returned %#v, want %#v", got, want)
|
|
}
|
|
if got, unwanted := tables, []string{"myothertable"}; containsAll(got, unwanted) {
|
|
t.Errorf("adminClient.Tables return %#v. unwanted %#v", got, unwanted)
|
|
}
|
|
|
|
tblConf := TableConf{
|
|
TableID: "conftable",
|
|
Families: map[string]GCPolicy{
|
|
"fam1": MaxVersionsPolicy(1),
|
|
"fam2": MaxVersionsPolicy(2),
|
|
},
|
|
}
|
|
if err := adminClient.CreateTableFromConf(ctx, &tblConf); err != nil {
|
|
t.Fatalf("Creating table from TableConf: %v", err)
|
|
}
|
|
defer adminClient.DeleteTable(ctx, tblConf.TableID)
|
|
|
|
tblInfo, err := adminClient.TableInfo(ctx, tblConf.TableID)
|
|
if err != nil {
|
|
t.Fatalf("Getting table info: %v", err)
|
|
}
|
|
sort.Strings(tblInfo.Families)
|
|
wantFams := []string{"fam1", "fam2"}
|
|
if !testutil.Equal(tblInfo.Families, wantFams) {
|
|
t.Errorf("Column family mismatch, got %v, want %v", tblInfo.Families, wantFams)
|
|
}
|
|
|
|
// Populate mytable and drop row ranges
|
|
if err = adminClient.CreateColumnFamily(ctx, "mytable", "cf"); err != nil {
|
|
t.Fatalf("Creating column family: %v", err)
|
|
}
|
|
|
|
client, err := testEnv.NewClient()
|
|
if err != nil {
|
|
t.Fatalf("NewClient: %v", err)
|
|
}
|
|
defer client.Close()
|
|
|
|
tbl := client.Open("mytable")
|
|
|
|
prefixes := []string{"a", "b", "c"}
|
|
for _, prefix := range prefixes {
|
|
for i := 0; i < 5; i++ {
|
|
mut := NewMutation()
|
|
mut.Set("cf", "col", 1000, []byte("1"))
|
|
if err := tbl.Apply(ctx, fmt.Sprintf("%v-%v", prefix, i), mut); err != nil {
|
|
t.Fatalf("Mutating row: %v", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
if err = adminClient.DropRowRange(ctx, "mytable", "a"); err != nil {
|
|
t.Errorf("DropRowRange a: %v", err)
|
|
}
|
|
if err = adminClient.DropRowRange(ctx, "mytable", "c"); err != nil {
|
|
t.Errorf("DropRowRange c: %v", err)
|
|
}
|
|
if err = adminClient.DropRowRange(ctx, "mytable", "x"); err != nil {
|
|
t.Errorf("DropRowRange x: %v", err)
|
|
}
|
|
|
|
var gotRowCount int
|
|
must(tbl.ReadRows(ctx, RowRange{}, func(row Row) bool {
|
|
gotRowCount++
|
|
if !strings.HasPrefix(row.Key(), "b") {
|
|
t.Errorf("Invalid row after dropping range: %v", row)
|
|
}
|
|
return true
|
|
}))
|
|
if gotRowCount != 5 {
|
|
t.Errorf("Invalid row count after dropping range: got %v, want %v", gotRowCount, 5)
|
|
}
|
|
}
|
|
|
|
func TestInstanceUpdate(t *testing.T) {
|
|
testEnv, err := NewIntegrationEnv()
|
|
if err != nil {
|
|
t.Fatalf("IntegrationEnv: %v", err)
|
|
}
|
|
defer testEnv.Close()
|
|
|
|
timeout := 2 * time.Second
|
|
if testEnv.Config().UseProd {
|
|
timeout = 5 * time.Minute
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
defer cancel()
|
|
|
|
adminClient, err := testEnv.NewAdminClient()
|
|
if err != nil {
|
|
t.Fatalf("NewAdminClient: %v", err)
|
|
}
|
|
|
|
defer adminClient.Close()
|
|
|
|
iAdminClient, err := testEnv.NewInstanceAdminClient()
|
|
if err != nil {
|
|
t.Fatalf("NewInstanceAdminClient: %v", err)
|
|
}
|
|
|
|
if iAdminClient == nil {
|
|
return
|
|
}
|
|
|
|
defer iAdminClient.Close()
|
|
|
|
iInfo, err := iAdminClient.InstanceInfo(ctx, adminClient.instance)
|
|
if err != nil {
|
|
t.Errorf("InstanceInfo: %v", err)
|
|
}
|
|
|
|
if iInfo.Name != adminClient.instance {
|
|
t.Errorf("InstanceInfo returned name %#v, want %#v", iInfo.Name, adminClient.instance)
|
|
}
|
|
|
|
if iInfo.DisplayName != adminClient.instance {
|
|
t.Errorf("InstanceInfo returned name %#v, want %#v", iInfo.Name, adminClient.instance)
|
|
}
|
|
|
|
const numNodes = 4
|
|
// update cluster nodes
|
|
if err := iAdminClient.UpdateCluster(ctx, adminClient.instance, testEnv.Config().Cluster, int32(numNodes)); err != nil {
|
|
t.Errorf("UpdateCluster: %v", err)
|
|
}
|
|
|
|
// get cluster after updating
|
|
cis, err := iAdminClient.GetCluster(ctx, adminClient.instance, testEnv.Config().Cluster)
|
|
if err != nil {
|
|
t.Errorf("GetCluster %v", err)
|
|
}
|
|
if cis.ServeNodes != int(numNodes) {
|
|
t.Errorf("ServeNodes returned %d, want %d", cis.ServeNodes, int(numNodes))
|
|
}
|
|
}
|
|
|
|
func TestAdminSnapshotIntegration(t *testing.T) {
|
|
testEnv, err := NewIntegrationEnv()
|
|
if err != nil {
|
|
t.Fatalf("IntegrationEnv: %v", err)
|
|
}
|
|
defer testEnv.Close()
|
|
|
|
if !testEnv.Config().UseProd {
|
|
t.Skip("emulator doesn't support snapshots")
|
|
}
|
|
|
|
timeout := 2 * time.Second
|
|
if testEnv.Config().UseProd {
|
|
timeout = 5 * time.Minute
|
|
}
|
|
ctx, _ := context.WithTimeout(context.Background(), timeout)
|
|
|
|
adminClient, err := testEnv.NewAdminClient()
|
|
if err != nil {
|
|
t.Fatalf("NewAdminClient: %v", err)
|
|
}
|
|
defer adminClient.Close()
|
|
|
|
table := testEnv.Config().Table
|
|
cluster := testEnv.Config().Cluster
|
|
|
|
list := func(cluster string) ([]*SnapshotInfo, error) {
|
|
infos := []*SnapshotInfo(nil)
|
|
|
|
it := adminClient.Snapshots(ctx, cluster)
|
|
for {
|
|
s, err := it.Next()
|
|
if err == iterator.Done {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
infos = append(infos, s)
|
|
}
|
|
return infos, err
|
|
}
|
|
|
|
// Delete the table at the end of the test. Schedule ahead of time
|
|
// in case the client fails
|
|
defer adminClient.DeleteTable(ctx, table)
|
|
|
|
if err := adminClient.CreateTable(ctx, table); err != nil {
|
|
t.Fatalf("Creating table: %v", err)
|
|
}
|
|
|
|
// Precondition: no snapshots
|
|
snapshots, err := list(cluster)
|
|
if err != nil {
|
|
t.Fatalf("Initial snapshot list: %v", err)
|
|
}
|
|
if got, want := len(snapshots), 0; got != want {
|
|
t.Fatalf("Initial snapshot list len: %d, want: %d", got, want)
|
|
}
|
|
|
|
// Create snapshot
|
|
defer adminClient.DeleteSnapshot(ctx, cluster, "mysnapshot")
|
|
|
|
if err = adminClient.SnapshotTable(ctx, table, cluster, "mysnapshot", 5*time.Hour); err != nil {
|
|
t.Fatalf("Creating snaphot: %v", err)
|
|
}
|
|
|
|
// List snapshot
|
|
snapshots, err = list(cluster)
|
|
if err != nil {
|
|
t.Fatalf("Listing snapshots: %v", err)
|
|
}
|
|
if got, want := len(snapshots), 1; got != want {
|
|
t.Fatalf("Listing snapshot count: %d, want: %d", got, want)
|
|
}
|
|
if got, want := snapshots[0].Name, "mysnapshot"; got != want {
|
|
t.Fatalf("Snapshot name: %s, want: %s", got, want)
|
|
}
|
|
if got, want := snapshots[0].SourceTable, table; got != want {
|
|
t.Fatalf("Snapshot SourceTable: %s, want: %s", got, want)
|
|
}
|
|
if got, want := snapshots[0].DeleteTime, snapshots[0].CreateTime.Add(5*time.Hour); math.Abs(got.Sub(want).Minutes()) > 1 {
|
|
t.Fatalf("Snapshot DeleteTime: %s, want: %s", got, want)
|
|
}
|
|
|
|
// Get snapshot
|
|
snapshot, err := adminClient.SnapshotInfo(ctx, cluster, "mysnapshot")
|
|
if err != nil {
|
|
t.Fatalf("SnapshotInfo: %v", snapshot)
|
|
}
|
|
if got, want := *snapshot, *snapshots[0]; got != want {
|
|
t.Fatalf("SnapshotInfo: %v, want: %v", got, want)
|
|
}
|
|
|
|
// Restore
|
|
restoredTable := table + "-restored"
|
|
defer adminClient.DeleteTable(ctx, restoredTable)
|
|
if err = adminClient.CreateTableFromSnapshot(ctx, restoredTable, cluster, "mysnapshot"); err != nil {
|
|
t.Fatalf("CreateTableFromSnapshot: %v", err)
|
|
}
|
|
if _, err := adminClient.TableInfo(ctx, restoredTable); err != nil {
|
|
t.Fatalf("Restored TableInfo: %v", err)
|
|
}
|
|
|
|
// Delete snapshot
|
|
if err = adminClient.DeleteSnapshot(ctx, cluster, "mysnapshot"); err != nil {
|
|
t.Fatalf("DeleteSnapshot: %v", err)
|
|
}
|
|
snapshots, err = list(cluster)
|
|
if err != nil {
|
|
t.Fatalf("List after Delete: %v", err)
|
|
}
|
|
if got, want := len(snapshots), 0; got != want {
|
|
t.Fatalf("List after delete len: %d, want: %d", got, want)
|
|
}
|
|
}
|
|
|
|
func TestGranularity(t *testing.T) {
|
|
testEnv, err := NewIntegrationEnv()
|
|
if err != nil {
|
|
t.Fatalf("IntegrationEnv: %v", err)
|
|
}
|
|
defer testEnv.Close()
|
|
|
|
timeout := 2 * time.Second
|
|
if testEnv.Config().UseProd {
|
|
timeout = 5 * time.Minute
|
|
}
|
|
ctx, _ := context.WithTimeout(context.Background(), timeout)
|
|
|
|
adminClient, err := testEnv.NewAdminClient()
|
|
if err != nil {
|
|
t.Fatalf("NewAdminClient: %v", err)
|
|
}
|
|
defer adminClient.Close()
|
|
|
|
list := func() []string {
|
|
tbls, err := adminClient.Tables(ctx)
|
|
if err != nil {
|
|
t.Fatalf("Fetching list of tables: %v", err)
|
|
}
|
|
sort.Strings(tbls)
|
|
return tbls
|
|
}
|
|
containsAll := func(got, want []string) bool {
|
|
gotSet := make(map[string]bool)
|
|
|
|
for _, s := range got {
|
|
gotSet[s] = true
|
|
}
|
|
for _, s := range want {
|
|
if !gotSet[s] {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
defer adminClient.DeleteTable(ctx, "mytable")
|
|
|
|
if err := adminClient.CreateTable(ctx, "mytable"); err != nil {
|
|
t.Fatalf("Creating table: %v", err)
|
|
}
|
|
|
|
tables := list()
|
|
if got, want := tables, []string{"mytable"}; !containsAll(got, want) {
|
|
t.Errorf("adminClient.Tables returned %#v, want %#v", got, want)
|
|
}
|
|
|
|
// calling ModifyColumnFamilies to check the granularity of table
|
|
prefix := adminClient.instancePrefix()
|
|
req := &btapb.ModifyColumnFamiliesRequest{
|
|
Name: prefix + "/tables/" + "mytable",
|
|
Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
|
|
Id: "cf",
|
|
Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Create{&btapb.ColumnFamily{}},
|
|
}},
|
|
}
|
|
table, err := adminClient.tClient.ModifyColumnFamilies(ctx, req)
|
|
if err != nil {
|
|
t.Fatalf("Creating column family: %v", err)
|
|
}
|
|
if table.Granularity != btapb.Table_TimestampGranularity(btapb.Table_MILLIS) {
|
|
t.Errorf("ModifyColumnFamilies returned granularity %#v, want %#v", table.Granularity, btapb.Table_TimestampGranularity(btapb.Table_MILLIS))
|
|
}
|
|
}
|
|
|
|
func TestInstanceAdminClient_AppProfile(t *testing.T) {
|
|
testEnv, err := NewIntegrationEnv()
|
|
if err != nil {
|
|
t.Fatalf("IntegrationEnv: %v", err)
|
|
}
|
|
defer testEnv.Close()
|
|
|
|
timeout := 2 * time.Second
|
|
if testEnv.Config().UseProd {
|
|
timeout = 5 * time.Minute
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
defer cancel()
|
|
|
|
adminClient, err := testEnv.NewAdminClient()
|
|
if err != nil {
|
|
t.Fatalf("NewAdminClient: %v", err)
|
|
}
|
|
defer adminClient.Close()
|
|
|
|
iAdminClient, err := testEnv.NewInstanceAdminClient()
|
|
if err != nil {
|
|
t.Fatalf("NewInstanceAdminClient: %v", err)
|
|
}
|
|
|
|
if iAdminClient == nil {
|
|
return
|
|
}
|
|
|
|
defer iAdminClient.Close()
|
|
profile := ProfileConf{
|
|
ProfileID: "app_profile1",
|
|
InstanceID: adminClient.instance,
|
|
ClusterID: testEnv.Config().Cluster,
|
|
Description: "creating new app profile 1",
|
|
RoutingPolicy: SingleClusterRouting,
|
|
}
|
|
|
|
createdProfile, err := iAdminClient.CreateAppProfile(ctx, profile)
|
|
if err != nil {
|
|
t.Fatalf("Creating app profile: %v", err)
|
|
|
|
}
|
|
|
|
gotProfile, err := iAdminClient.GetAppProfile(ctx, adminClient.instance, "app_profile1")
|
|
|
|
if err != nil {
|
|
t.Fatalf("Get app profile: %v", err)
|
|
}
|
|
|
|
if !proto.Equal(createdProfile, gotProfile) {
|
|
t.Fatalf("created profile: %s, got profile: %s", createdProfile.Name, gotProfile.Name)
|
|
|
|
}
|
|
|
|
list := func(instanceID string) ([]*btapb.AppProfile, error) {
|
|
profiles := []*btapb.AppProfile(nil)
|
|
|
|
it := iAdminClient.ListAppProfiles(ctx, instanceID)
|
|
for {
|
|
s, err := it.Next()
|
|
if err == iterator.Done {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
profiles = append(profiles, s)
|
|
}
|
|
return profiles, err
|
|
}
|
|
|
|
profiles, err := list(adminClient.instance)
|
|
if err != nil {
|
|
t.Fatalf("List app profile: %v", err)
|
|
}
|
|
|
|
if got, want := len(profiles), 1; got != want {
|
|
t.Fatalf("Initial app profile list len: %d, want: %d", got, want)
|
|
}
|
|
|
|
for _, test := range []struct {
|
|
desc string
|
|
uattrs ProfileAttrsToUpdate
|
|
want *btapb.AppProfile // nil means error
|
|
}{
|
|
{
|
|
desc: "empty update",
|
|
uattrs: ProfileAttrsToUpdate{},
|
|
want: nil,
|
|
},
|
|
|
|
{
|
|
desc: "empty description update",
|
|
uattrs: ProfileAttrsToUpdate{Description: ""},
|
|
want: &btapb.AppProfile{
|
|
Name: gotProfile.Name,
|
|
Description: "",
|
|
RoutingPolicy: gotProfile.RoutingPolicy,
|
|
Etag: gotProfile.Etag},
|
|
},
|
|
{
|
|
desc: "routing update",
|
|
uattrs: ProfileAttrsToUpdate{
|
|
RoutingPolicy: SingleClusterRouting,
|
|
ClusterID: testEnv.Config().Cluster,
|
|
},
|
|
want: &btapb.AppProfile{
|
|
Name: gotProfile.Name,
|
|
Description: "",
|
|
Etag: gotProfile.Etag,
|
|
RoutingPolicy: &btapb.AppProfile_SingleClusterRouting_{
|
|
SingleClusterRouting: &btapb.AppProfile_SingleClusterRouting{
|
|
ClusterId: testEnv.Config().Cluster,
|
|
}},
|
|
},
|
|
},
|
|
} {
|
|
err = iAdminClient.UpdateAppProfile(ctx, adminClient.instance, "app_profile1", test.uattrs)
|
|
if err != nil {
|
|
if test.want != nil {
|
|
t.Errorf("%s: %v", test.desc, err)
|
|
}
|
|
continue
|
|
}
|
|
if err == nil && test.want == nil {
|
|
t.Errorf("%s: got nil, want error", test.desc)
|
|
continue
|
|
}
|
|
|
|
got, _ := iAdminClient.GetAppProfile(ctx, adminClient.instance, "app_profile1")
|
|
|
|
if !proto.Equal(got, test.want) {
|
|
t.Fatalf("%s : got profile : %v, want profile: %v", test.desc, gotProfile, test.want)
|
|
}
|
|
|
|
}
|
|
|
|
err = iAdminClient.DeleteAppProfile(ctx, adminClient.instance, "app_profile1")
|
|
if err != nil {
|
|
t.Fatalf("Delete app profile: %v", err)
|
|
}
|
|
|
|
}
|