Skip to content

Commit b680164

Browse files
shantstepanianshantstepanian
committed
feat(bigtable): adding node scaling factor
1 parent 587b5cc commit b680164

3 files changed

Lines changed: 148 additions & 23 deletions

File tree

‎bigtable/admin.go‎

Lines changed: 69 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1238,6 +1238,12 @@ type InstanceConf struct {
12381238
// AutoscalingConfig configures the autoscaling properties on the cluster
12391239
// created with the instance. It is optional.
12401240
AutoscalingConfig *AutoscalingConfig
1241+
1242+
// NodeScalingFactor controls the scaling factor of the cluster (i.e. the
1243+
// increment in which NumNodes can be set). Node scaling delivers better
1244+
// latency and more throughput by removing node boundaries. It is optional,
1245+
// with the default being 1X.
1246+
NodeScalingFactor NodeScalingFactor
12411247
}
12421248

12431249
// InstanceWithClustersConfig contains the information necessary to create an Instance
@@ -1267,6 +1273,7 @@ func (iac *InstanceAdminClient) CreateInstance(ctx context.Context, conf *Instan
12671273
NumNodes: conf.NumNodes,
12681274
StorageType: conf.StorageType,
12691275
AutoscalingConfig: conf.AutoscalingConfig,
1276+
NodeScalingFactor: conf.NodeScalingFactor,
12701277
},
12711278
},
12721279
}
@@ -1507,6 +1514,44 @@ func (a *AutoscalingConfig) proto() *btapb.Cluster_ClusterAutoscalingConfig {
15071514
}
15081515
}
15091516

1517+
// NodeScalingFactor controls the scaling factor of the cluster (i.e. the
1518+
// increment in which NumNodes can be set). Node scaling delivers better
1519+
// latency and more throughput by removing node boundaries.
1520+
type NodeScalingFactor int32
1521+
1522+
const (
1523+
// NodeScalingFactorUnspecified default to 1X.
1524+
NodeScalingFactorUnspecified NodeScalingFactor = iota
1525+
// NodeScalingFactor1X runs the cluster with a scaling factor of 1.
1526+
NodeScalingFactor1X
1527+
// NodeScalingFactor2X runs the cluster with a scaling factor of 2.
1528+
// All node count values must be in increments of 2 with this scaling
1529+
// factor enabled, otherwise an INVALID_ARGUMENT error will be returned.
1530+
NodeScalingFactor2X
1531+
)
1532+
1533+
func (nsf NodeScalingFactor) proto() btapb.Cluster_NodeScalingFactor {
1534+
switch nsf {
1535+
case NodeScalingFactor1X:
1536+
return btapb.Cluster_NODE_SCALING_FACTOR_1X
1537+
case NodeScalingFactor2X:
1538+
return btapb.Cluster_NODE_SCALING_FACTOR_2X
1539+
default:
1540+
return btapb.Cluster_NODE_SCALING_FACTOR_UNSPECIFIED
1541+
}
1542+
}
1543+
1544+
func nodeScalingFactorFromProto(nsf btapb.Cluster_NodeScalingFactor) NodeScalingFactor {
1545+
switch nsf {
1546+
case btapb.Cluster_NODE_SCALING_FACTOR_1X:
1547+
return NodeScalingFactor1X
1548+
case btapb.Cluster_NODE_SCALING_FACTOR_2X:
1549+
return NodeScalingFactor2X
1550+
default:
1551+
return NodeScalingFactorUnspecified
1552+
}
1553+
}
1554+
15101555
// ClusterConfig contains the information necessary to create a cluster
15111556
type ClusterConfig struct {
15121557
// InstanceID specifies the unique name of the instance. Required.
@@ -1548,6 +1593,12 @@ type ClusterConfig struct {
15481593
// AutoscalingConfig configures the autoscaling properties on a cluster.
15491594
// One of NumNodes or AutoscalingConfig is required.
15501595
AutoscalingConfig *AutoscalingConfig
1596+
1597+
// NodeScalingFactor controls the scaling factor of the cluster (i.e. the
1598+
// increment in which NumNodes can be set). Node scaling delivers better
1599+
// latency and more throughput by removing node boundaries. It is optional,
1600+
// with the default being 1X.
1601+
NodeScalingFactor NodeScalingFactor
15511602
}
15521603

15531604
func (cc *ClusterConfig) proto(project string) *btapb.Cluster {
@@ -1558,6 +1609,7 @@ func (cc *ClusterConfig) proto(project string) *btapb.Cluster {
15581609
EncryptionConfig: &btapb.Cluster_EncryptionConfig{
15591610
KmsKeyName: cc.KMSKeyName,
15601611
},
1612+
NodeScalingFactor: cc.NodeScalingFactor.proto(),
15611613
}
15621614

15631615
if asc := cc.AutoscalingConfig; asc != nil {
@@ -1592,6 +1644,9 @@ type ClusterInfo struct {
15921644

15931645
// AutoscalingConfig are the configured values for a cluster.
15941646
AutoscalingConfig *AutoscalingConfig
1647+
1648+
// NodeScalingFactor controls the scaling factor of the cluster.
1649+
NodeScalingFactor NodeScalingFactor
15951650
}
15961651

15971652
// CreateCluster creates a new cluster in an instance.
@@ -1695,12 +1750,13 @@ func (iac *InstanceAdminClient) Clusters(ctx context.Context, instanceID string)
16951750
kmsKeyName = c.EncryptionConfig.KmsKeyName
16961751
}
16971752
ci := &ClusterInfo{
1698-
Name: nameParts[len(nameParts)-1],
1699-
Zone: locParts[len(locParts)-1],
1700-
ServeNodes: int(c.ServeNodes),
1701-
State: c.State.String(),
1702-
StorageType: storageTypeFromProto(c.DefaultStorageType),
1703-
KMSKeyName: kmsKeyName,
1753+
Name: nameParts[len(nameParts)-1],
1754+
Zone: locParts[len(locParts)-1],
1755+
ServeNodes: int(c.ServeNodes),
1756+
State: c.State.String(),
1757+
StorageType: storageTypeFromProto(c.DefaultStorageType),
1758+
KMSKeyName: kmsKeyName,
1759+
NodeScalingFactor: nodeScalingFactorFromProto(c.NodeScalingFactor),
17041760
}
17051761
if cfg := c.GetClusterConfig(); cfg != nil {
17061762
if asc := fromClusterConfigProto(cfg); asc != nil {
@@ -1740,12 +1796,13 @@ func (iac *InstanceAdminClient) GetCluster(ctx context.Context, instanceID, clus
17401796
nameParts := strings.Split(c.Name, "/")
17411797
locParts := strings.Split(c.Location, "/")
17421798
ci := &ClusterInfo{
1743-
Name: nameParts[len(nameParts)-1],
1744-
Zone: locParts[len(locParts)-1],
1745-
ServeNodes: int(c.ServeNodes),
1746-
State: c.State.String(),
1747-
StorageType: storageTypeFromProto(c.DefaultStorageType),
1748-
KMSKeyName: kmsKeyName,
1799+
Name: nameParts[len(nameParts)-1],
1800+
Zone: locParts[len(locParts)-1],
1801+
ServeNodes: int(c.ServeNodes),
1802+
State: c.State.String(),
1803+
StorageType: storageTypeFromProto(c.DefaultStorageType),
1804+
KMSKeyName: kmsKeyName,
1805+
NodeScalingFactor: nodeScalingFactorFromProto(c.NodeScalingFactor),
17491806
}
17501807
// Use type assertion to handle protobuf oneof type
17511808
if cfg := c.GetClusterConfig(); cfg != nil {

‎bigtable/admin_test.go‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -842,6 +842,7 @@ func TestInstanceAdmin_GetCluster(t *testing.T) {
842842
Location: ".../us-central1-a",
843843
State: btapb.Cluster_READY,
844844
DefaultStorageType: btapb.StorageType_SSD,
845+
NodeScalingFactor: btapb.Cluster_NODE_SCALING_FACTOR_1X,
845846
},
846847
wantConfig: nil,
847848
},
@@ -852,6 +853,7 @@ func TestInstanceAdmin_GetCluster(t *testing.T) {
852853
Location: ".../us-central1-a",
853854
State: btapb.Cluster_READY,
854855
DefaultStorageType: btapb.StorageType_SSD,
856+
NodeScalingFactor: btapb.Cluster_NODE_SCALING_FACTOR_1X,
855857
Config: &btapb.Cluster_ClusterConfig_{
856858
ClusterConfig: &btapb.Cluster_ClusterConfig{
857859
ClusterAutoscalingConfig: &btapb.Cluster_ClusterAutoscalingConfig{
@@ -900,6 +902,7 @@ func TestInstanceAdmin_Clusters(t *testing.T) {
900902
Location: ".../us-central1-a",
901903
State: btapb.Cluster_READY,
902904
DefaultStorageType: btapb.StorageType_SSD,
905+
NodeScalingFactor: btapb.Cluster_NODE_SCALING_FACTOR_1X,
903906
},
904907
wantConfig: nil,
905908
},
@@ -910,6 +913,7 @@ func TestInstanceAdmin_Clusters(t *testing.T) {
910913
Location: ".../us-central1-a",
911914
State: btapb.Cluster_READY,
912915
DefaultStorageType: btapb.StorageType_SSD,
916+
NodeScalingFactor: btapb.Cluster_NODE_SCALING_FACTOR_1X,
913917
Config: &btapb.Cluster_ClusterConfig_{
914918
ClusterConfig: &btapb.Cluster_ClusterConfig{
915919
ClusterAutoscalingConfig: &btapb.Cluster_ClusterAutoscalingConfig{
@@ -1027,6 +1031,7 @@ func TestInstanceAdmin_CreateInstance_WithAutoscaling(t *testing.T) {
10271031
ClusterId: "mycluster",
10281032
Zone: "us-central1-a",
10291033
StorageType: SSD,
1034+
NodeScalingFactor: NodeScalingFactor1X,
10301035
AutoscalingConfig: &AutoscalingConfig{MinNodes: 1, MaxNodes: 2, CPUTargetPercent: 10, StorageUtilizationPerNode: 3000},
10311036
})
10321037
if err != nil {
@@ -1053,13 +1058,14 @@ func TestInstanceAdmin_CreateInstance_WithAutoscaling(t *testing.T) {
10531058
}
10541059

10551060
err = c.CreateInstance(context.Background(), &InstanceConf{
1056-
InstanceId: "myinst",
1057-
DisplayName: "myinst",
1058-
InstanceType: PRODUCTION,
1059-
ClusterId: "mycluster",
1060-
Zone: "us-central1-a",
1061-
StorageType: SSD,
1062-
NumNodes: 1,
1061+
InstanceId: "myinst",
1062+
DisplayName: "myinst",
1063+
InstanceType: PRODUCTION,
1064+
ClusterId: "mycluster",
1065+
Zone: "us-central1-a",
1066+
StorageType: SSD,
1067+
NodeScalingFactor: NodeScalingFactor1X,
1068+
NumNodes: 1,
10631069
})
10641070
if err != nil {
10651071
t.Fatalf("CreateInstance failed: %v", err)
@@ -1085,6 +1091,7 @@ func TestInstanceAdmin_CreateInstanceWithClusters_WithAutoscaling(t *testing.T)
10851091
ClusterID: "mycluster",
10861092
Zone: "us-central1-a",
10871093
StorageType: SSD,
1094+
NodeScalingFactor: NodeScalingFactor1X,
10881095
AutoscalingConfig: &AutoscalingConfig{MinNodes: 1, MaxNodes: 2, CPUTargetPercent: 10, StorageUtilizationPerNode: 3000},
10891096
},
10901097
},
@@ -1122,6 +1129,7 @@ func TestInstanceAdmin_CreateCluster_WithAutoscaling(t *testing.T) {
11221129
Zone: "us-central1-a",
11231130
StorageType: SSD,
11241131
AutoscalingConfig: &AutoscalingConfig{MinNodes: 1, MaxNodes: 2, CPUTargetPercent: 10, StorageUtilizationPerNode: 3000},
1132+
NodeScalingFactor: NodeScalingFactor1X,
11251133
})
11261134
if err != nil {
11271135
t.Fatalf("CreateCluster failed: %v", err)
@@ -1151,10 +1159,11 @@ func TestInstanceAdmin_CreateCluster_WithAutoscaling(t *testing.T) {
11511159
}
11521160

11531161
err = c.CreateCluster(context.Background(), &ClusterConfig{
1154-
ClusterID: "mycluster",
1155-
Zone: "us-central1-a",
1156-
StorageType: SSD,
1157-
NumNodes: 1,
1162+
ClusterID: "mycluster",
1163+
Zone: "us-central1-a",
1164+
StorageType: SSD,
1165+
NumNodes: 1,
1166+
NodeScalingFactor: NodeScalingFactor1X,
11581167
})
11591168
if err != nil {
11601169
t.Fatalf("CreateCluster failed: %v", err)

‎bigtable/integration_test.go‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3631,6 +3631,65 @@ func TestIntegration_InstanceAdminClient_AppProfile(t *testing.T) {
36313631
}
36323632
}
36333633

3634+
func TestIntegration_NodeScalingFactor(t *testing.T) {
3635+
if instanceToCreate == "" {
3636+
t.Skip("instanceToCreate not set, skipping instance update testing")
3637+
}
3638+
instanceToCreate += "5"
3639+
3640+
testEnv, err := NewIntegrationEnv()
3641+
if err != nil {
3642+
t.Fatalf("IntegrationEnv: %v", err)
3643+
}
3644+
defer testEnv.Close()
3645+
3646+
if !testEnv.Config().UseProd {
3647+
t.Skip("emulator doesn't support instance creation")
3648+
}
3649+
3650+
timeout := 10 * time.Minute
3651+
ctx, cancel := context.WithTimeout(context.Background(), timeout)
3652+
defer cancel()
3653+
3654+
iAdminClient, err := testEnv.NewInstanceAdminClient()
3655+
if err != nil {
3656+
t.Fatalf("NewInstanceAdminClient: %v", err)
3657+
}
3658+
defer iAdminClient.Close()
3659+
3660+
clusterID := instanceToCreate + "-cluster"
3661+
wantNodeScalingFactor := NodeScalingFactor2X
3662+
3663+
t.Log("creating an instance with node scaling factor")
3664+
conf := &InstanceWithClustersConfig{
3665+
InstanceID: instanceToCreate,
3666+
DisplayName: "test instance",
3667+
Clusters: []ClusterConfig{
3668+
{
3669+
ClusterID: clusterID,
3670+
NumNodes: 2,
3671+
NodeScalingFactor: wantNodeScalingFactor,
3672+
Zone: instanceToCreateZone,
3673+
},
3674+
},
3675+
}
3676+
defer iAdminClient.DeleteInstance(ctx, instanceToCreate)
3677+
err = retry(func() error { return iAdminClient.CreateInstanceWithClusters(ctx, conf) },
3678+
func() error { return iAdminClient.DeleteInstance(ctx, conf.InstanceID) })
3679+
if err != nil {
3680+
t.Fatalf("CreateInstanceWithClusters: %v", err)
3681+
}
3682+
3683+
cluster, err := iAdminClient.GetCluster(ctx, instanceToCreate, clusterID)
3684+
if err != nil {
3685+
t.Fatalf("GetCluster: %v", err)
3686+
}
3687+
3688+
if gotNodeScalingFactor := cluster.NodeScalingFactor; gotNodeScalingFactor != wantNodeScalingFactor {
3689+
t.Fatalf("NodeScalingFactor: got: %v, want: %v", gotNodeScalingFactor, wantNodeScalingFactor)
3690+
}
3691+
}
3692+
36343693
func TestIntegration_InstanceUpdate(t *testing.T) {
36353694
testEnv, err := NewIntegrationEnv()
36363695
if err != nil {

0 commit comments

Comments
 (0)