Skip to content

Commit 238ac1c

Browse files
bhshkhbhshkh
authored
feat(bigtable): Hot backups (#11215)
* feat(bigtable): Hot backups * test(bigtable): Add integration tests for update * refactoring for usability * 2 separate methods to update and remove hts time
1 parent aa48838 commit 238ac1c

3 files changed

Lines changed: 313 additions & 43 deletions

File tree

‎bigtable/admin.go‎

Lines changed: 145 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ import (
4747
const adminAddr = "bigtableadmin.googleapis.com:443"
4848
const mtlsAdminAddr = "bigtableadmin.mtls.googleapis.com:443"
4949

50+
var errExpiryMissing = errors.New("WithExpiry is a required option")
51+
5052
// ErrPartiallyUnavailable is returned when some locations (clusters) are
5153
// unavailable. Both partial results (retrieved from available locations)
5254
// and the error are returned when this exception occurred.
@@ -2150,13 +2152,68 @@ func (ac *AdminClient) RestoreTableFrom(ctx context.Context, sourceInstance, tab
21502152
return longrunning.InternalNewOperation(ac.lroClient, op).Wait(ctx, &resp)
21512153
}
21522154

2155+
type backupOptions struct {
2156+
backupType *BackupType
2157+
hotToStandardTime *time.Time
2158+
expireTime *time.Time
2159+
}
2160+
2161+
// BackupOption can be used to specify parameters for backup operations.
2162+
type BackupOption func(*backupOptions)
2163+
2164+
// WithHotToStandardBackup option can be used to create backup with
2165+
// type [BackupTypeHot] and specify time at which the hot backup will be
2166+
// converted to a standard backup. Once the 'hotToStandardTime' has passed,
2167+
// Cloud Bigtable will convert the hot backup to a standard backup.
2168+
// This value must be greater than the backup creation time by at least 24 hours
2169+
func WithHotToStandardBackup(hotToStandardTime time.Time) BackupOption {
2170+
return func(bo *backupOptions) {
2171+
btHot := BackupTypeHot
2172+
bo.backupType = &btHot
2173+
bo.hotToStandardTime = &hotToStandardTime
2174+
}
2175+
}
2176+
2177+
// WithExpiry option can be used to create backup
2178+
// that expires after time 'expireTime'.
2179+
// Once the 'expireTime' has passed, Cloud Bigtable will delete the backup.
2180+
func WithExpiry(expireTime time.Time) BackupOption {
2181+
return func(bo *backupOptions) {
2182+
bo.expireTime = &expireTime
2183+
}
2184+
}
2185+
2186+
// WithHotBackup option can be used to create backup
2187+
// with type [BackupTypeHot]
2188+
func WithHotBackup() BackupOption {
2189+
return func(bo *backupOptions) {
2190+
btHot := BackupTypeHot
2191+
bo.backupType = &btHot
2192+
}
2193+
}
2194+
21532195
// CreateBackup creates a new backup in the specified cluster from the
21542196
// specified source table with the user-provided expire time.
21552197
func (ac *AdminClient) CreateBackup(ctx context.Context, table, cluster, backup string, expireTime time.Time) error {
2198+
return ac.CreateBackupWithOptions(ctx, table, cluster, backup, WithExpiry(expireTime))
2199+
}
2200+
2201+
// CreateBackupWithOptions is similar to CreateBackup but lets the user specify additional options.
2202+
func (ac *AdminClient) CreateBackupWithOptions(ctx context.Context, table, cluster, backup string, opts ...BackupOption) error {
21562203
ctx = mergeOutgoingMetadata(ctx, ac.md)
21572204
prefix := ac.instancePrefix()
21582205

2159-
parsedExpireTime := timestamppb.New(expireTime)
2206+
o := backupOptions{}
2207+
for _, opt := range opts {
2208+
if opt != nil {
2209+
opt(&o)
2210+
}
2211+
}
2212+
2213+
if o.expireTime == nil {
2214+
return errExpiryMissing
2215+
}
2216+
parsedExpireTime := timestamppb.New(*o.expireTime)
21602217

21612218
req := &btapb.CreateBackupRequest{
21622219
Parent: prefix + "/clusters/" + cluster,
@@ -2167,6 +2224,12 @@ func (ac *AdminClient) CreateBackup(ctx context.Context, table, cluster, backup
21672224
},
21682225
}
21692226

2227+
if o.backupType != nil {
2228+
req.Backup.BackupType = btapb.Backup_BackupType(*o.backupType)
2229+
}
2230+
if o.hotToStandardTime != nil {
2231+
req.Backup.HotToStandardTime = timestamppb.New(*o.hotToStandardTime)
2232+
}
21702233
op, err := ac.tClient.CreateBackup(ctx, req)
21712234
if err != nil {
21722235
return err
@@ -2263,17 +2326,29 @@ func newBackupInfo(backup *btapb.Backup) (*BackupInfo, error) {
22632326
return nil, fmt.Errorf("invalid expireTime: %v", err)
22642327
}
22652328
expireTime := backup.GetExpireTime().AsTime()
2329+
2330+
var htsTimePtr *time.Time
2331+
if backup.GetHotToStandardTime() != nil {
2332+
if err := backup.GetHotToStandardTime().CheckValid(); err != nil {
2333+
return nil, fmt.Errorf("invalid HotToStandardTime: %v", err)
2334+
}
2335+
htsTime := backup.GetHotToStandardTime().AsTime()
2336+
htsTimePtr = &htsTime
2337+
}
2338+
22662339
encryptionInfo := newEncryptionInfo(backup.EncryptionInfo)
22672340
bi := BackupInfo{
2268-
Name: name,
2269-
SourceTable: tableID,
2270-
SourceBackup: backup.SourceBackup,
2271-
SizeBytes: backup.SizeBytes,
2272-
StartTime: startTime,
2273-
EndTime: endTime,
2274-
ExpireTime: expireTime,
2275-
State: backup.State.String(),
2276-
EncryptionInfo: encryptionInfo,
2341+
Name: name,
2342+
SourceTable: tableID,
2343+
SourceBackup: backup.SourceBackup,
2344+
SizeBytes: backup.SizeBytes,
2345+
StartTime: startTime,
2346+
EndTime: endTime,
2347+
ExpireTime: expireTime,
2348+
State: backup.State.String(),
2349+
EncryptionInfo: encryptionInfo,
2350+
BackupType: BackupType(backup.GetBackupType()),
2351+
HotToStandardTime: htsTimePtr,
22772352
}
22782353

22792354
return &bi, nil
@@ -2303,6 +2378,25 @@ func (it *BackupIterator) Next() (*BackupInfo, error) {
23032378
return item, nil
23042379
}
23052380

2381+
// BackupType denotes the type of the backup.
2382+
type BackupType int32
2383+
2384+
const (
2385+
// BackupTypeUnspecified denotes that backup type has not been specified.
2386+
BackupTypeUnspecified BackupType = 0
2387+
2388+
// BackupTypeStandard is the default type for Cloud Bigtable managed backups. Supported for
2389+
// backups created in both HDD and SSD instances. Requires optimization when
2390+
// restored to a table in an SSD instance.
2391+
BackupTypeStandard BackupType = 1
2392+
2393+
// BackupTypeHot is a backup type with faster restore to SSD performance. Only supported for
2394+
// backups created in SSD instances. A new SSD table restored from a hot
2395+
// backup reaches production performance more quickly than a standard
2396+
// backup.
2397+
BackupTypeHot BackupType = 2
2398+
)
2399+
23062400
// BackupInfo contains backup metadata. This struct is read-only.
23072401
type BackupInfo struct {
23082402
Name string
@@ -2314,6 +2408,15 @@ type BackupInfo struct {
23142408
ExpireTime time.Time
23152409
State string
23162410
EncryptionInfo *EncryptionInfo
2411+
BackupType BackupType
2412+
2413+
// The time at which the hot backup will be converted to a standard backup.
2414+
// Once the `hot_to_standard_time` has passed, Cloud Bigtable will convert the
2415+
// hot backup to a standard backup. This value must be greater than the backup
2416+
// creation time by at least 24 hours
2417+
//
2418+
// This field only applies for hot backups.
2419+
HotToStandardTime *time.Time
23172420
}
23182421

23192422
// BackupInfo gets backup metadata.
@@ -2371,6 +2474,38 @@ func (ac *AdminClient) UpdateBackup(ctx context.Context, cluster, backup string,
23712474
return err
23722475
}
23732476

2477+
// UpdateBackupHotToStandardTime updates the HotToStandardTime of a hot backup.
2478+
func (ac *AdminClient) UpdateBackupHotToStandardTime(ctx context.Context, cluster, backup string, hotToStandardTime time.Time) error {
2479+
return ac.updateBackupHotToStandardTime(ctx, cluster, backup, &hotToStandardTime)
2480+
}
2481+
2482+
// UpdateBackupRemoveHotToStandardTime removes the HotToStandardTime of a hot backup.
2483+
func (ac *AdminClient) UpdateBackupRemoveHotToStandardTime(ctx context.Context, cluster, backup string) error {
2484+
return ac.updateBackupHotToStandardTime(ctx, cluster, backup, nil)
2485+
}
2486+
2487+
func (ac *AdminClient) updateBackupHotToStandardTime(ctx context.Context, cluster, backup string, hotToStandardTime *time.Time) error {
2488+
ctx = mergeOutgoingMetadata(ctx, ac.md)
2489+
backupPath := ac.backupPath(cluster, ac.instance, backup)
2490+
2491+
updateMask := &field_mask.FieldMask{}
2492+
updateMask.Paths = append(updateMask.Paths, "hot_to_standard_time")
2493+
2494+
req := &btapb.UpdateBackupRequest{
2495+
Backup: &btapb.Backup{
2496+
Name: backupPath,
2497+
},
2498+
UpdateMask: updateMask,
2499+
}
2500+
2501+
if hotToStandardTime != nil {
2502+
req.Backup.HotToStandardTime = timestamppb.New(*hotToStandardTime)
2503+
}
2504+
2505+
_, err := ac.tClient.UpdateBackup(ctx, req)
2506+
return err
2507+
}
2508+
23742509
// AuthorizedViewConf contains information about an authorized view.
23752510
type AuthorizedViewConf struct {
23762511
TableID string

‎bigtable/admin_test.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,16 @@ func TestTableAdmin_CreateTableFromConf_AutomatedBackupPolicy_Valid(t *testing.T
205205
}
206206
}
207207

208+
func TestTableAdmin_CreateBackupWithOptions_NoExpiryTime(t *testing.T) {
209+
mock := &mockTableAdminClock{}
210+
c := setupTableClient(t, mock)
211+
212+
err := c.CreateBackupWithOptions(context.Background(), "table", "cluster", "backup-01")
213+
if err == nil || !errors.Is(err, errExpiryMissing) {
214+
t.Errorf("CreateBackupWithOptions got: %v, want: %v error", err, errExpiryMissing)
215+
}
216+
}
217+
208218
func TestTableAdmin_CopyBackup_ErrorFromClient(t *testing.T) {
209219
mock := &mockTableAdminClock{}
210220
c := setupTableClient(t, mock)

0 commit comments

Comments
 (0)