Skip to content

Commit 985deb2

Browse files
BrennaEppBrennaEpp
authored
feat(storage): support for soft delete policies and restore (#9520)
1 parent bbfc0ac commit 985deb2

8 files changed

Lines changed: 437 additions & 47 deletions

File tree

‎storage/bucket.go‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -479,6 +479,13 @@ type BucketAttrs struct {
479479
// cannot be modified once the bucket is created.
480480
// ObjectRetention cannot be configured or reported through the gRPC API.
481481
ObjectRetentionMode string
482+
483+
// SoftDeletePolicy contains the bucket's soft delete policy, which defines
484+
// the period of time that soft-deleted objects will be retained, and cannot
485+
// be permanently deleted. By default, new buckets will be created with a
486+
// 7 day retention duration. In order to fully disable soft delete, you need
487+
// to set a policy with a RetentionDuration of 0.
488+
SoftDeletePolicy *SoftDeletePolicy
482489
}
483490

484491
// BucketPolicyOnly is an alias for UniformBucketLevelAccess.
@@ -766,6 +773,19 @@ type Autoclass struct {
766773
TerminalStorageClassUpdateTime time.Time
767774
}
768775

776+
// SoftDeletePolicy contains the bucket's soft delete policy, which defines the
777+
// period of time that soft-deleted objects will be retained, and cannot be
778+
// permanently deleted.
779+
type SoftDeletePolicy struct {
780+
// EffectiveTime indicates the time from which the policy, or one with a
781+
// greater retention, was effective. This field is read-only.
782+
EffectiveTime time.Time
783+
784+
// RetentionDuration is the amount of time that soft-deleted objects in the
785+
// bucket will be retained and cannot be permanently deleted.
786+
RetentionDuration time.Duration
787+
}
788+
769789
func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
770790
if b == nil {
771791
return nil, nil
@@ -803,6 +823,7 @@ func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
803823
RPO: toRPO(b),
804824
CustomPlacementConfig: customPlacementFromRaw(b.CustomPlacementConfig),
805825
Autoclass: toAutoclassFromRaw(b.Autoclass),
826+
SoftDeletePolicy: toSoftDeletePolicyFromRaw(b.SoftDeletePolicy),
806827
}, nil
807828
}
808829

@@ -836,6 +857,7 @@ func newBucketFromProto(b *storagepb.Bucket) *BucketAttrs {
836857
CustomPlacementConfig: customPlacementFromProto(b.GetCustomPlacementConfig()),
837858
ProjectNumber: parseProjectNumber(b.GetProject()), // this can return 0 the project resource name is ID based
838859
Autoclass: toAutoclassFromProto(b.GetAutoclass()),
860+
SoftDeletePolicy: toSoftDeletePolicyFromProto(b.SoftDeletePolicy),
839861
}
840862
}
841863

@@ -891,6 +913,7 @@ func (b *BucketAttrs) toRawBucket() *raw.Bucket {
891913
Rpo: b.RPO.String(),
892914
CustomPlacementConfig: b.CustomPlacementConfig.toRawCustomPlacement(),
893915
Autoclass: b.Autoclass.toRawAutoclass(),
916+
SoftDeletePolicy: b.SoftDeletePolicy.toRawSoftDeletePolicy(),
894917
}
895918
}
896919

@@ -951,6 +974,7 @@ func (b *BucketAttrs) toProtoBucket() *storagepb.Bucket {
951974
Rpo: b.RPO.String(),
952975
CustomPlacementConfig: b.CustomPlacementConfig.toProtoCustomPlacement(),
953976
Autoclass: b.Autoclass.toProtoAutoclass(),
977+
SoftDeletePolicy: b.SoftDeletePolicy.toProtoSoftDeletePolicy(),
954978
}
955979
}
956980

@@ -1032,6 +1056,7 @@ func (ua *BucketAttrsToUpdate) toProtoBucket() *storagepb.Bucket {
10321056
IamConfig: bktIAM,
10331057
Rpo: ua.RPO.String(),
10341058
Autoclass: ua.Autoclass.toProtoAutoclass(),
1059+
SoftDeletePolicy: ua.SoftDeletePolicy.toProtoSoftDeletePolicy(),
10351060
Labels: ua.setLabels,
10361061
}
10371062
}
@@ -1152,6 +1177,9 @@ type BucketAttrsToUpdate struct {
11521177
// See https://cloud.google.com/storage/docs/using-autoclass for more information.
11531178
Autoclass *Autoclass
11541179

1180+
// If set, updates the soft delete policy of the bucket.
1181+
SoftDeletePolicy *SoftDeletePolicy
1182+
11551183
// acl is the list of access control rules on the bucket.
11561184
// It is unexported and only used internally by the gRPC client.
11571185
// Library users should use ACLHandle methods directly.
@@ -1273,6 +1301,14 @@ func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket {
12731301
}
12741302
rb.ForceSendFields = append(rb.ForceSendFields, "Autoclass")
12751303
}
1304+
if ua.SoftDeletePolicy != nil {
1305+
if ua.SoftDeletePolicy.RetentionDuration == 0 {
1306+
rb.NullFields = append(rb.NullFields, "SoftDeletePolicy")
1307+
rb.SoftDeletePolicy = nil
1308+
} else {
1309+
rb.SoftDeletePolicy = ua.SoftDeletePolicy.toRawSoftDeletePolicy()
1310+
}
1311+
}
12761312
if ua.PredefinedACL != "" {
12771313
// Clear ACL or the call will fail.
12781314
rb.Acl = nil
@@ -2053,6 +2089,53 @@ func toAutoclassFromProto(a *storagepb.Bucket_Autoclass) *Autoclass {
20532089
}
20542090
}
20552091

2092+
func (p *SoftDeletePolicy) toRawSoftDeletePolicy() *raw.BucketSoftDeletePolicy {
2093+
if p == nil {
2094+
return nil
2095+
}
2096+
// Excluding read only field EffectiveTime.
2097+
return &raw.BucketSoftDeletePolicy{
2098+
RetentionDurationSeconds: int64(p.RetentionDuration.Seconds()),
2099+
}
2100+
}
2101+
2102+
func (p *SoftDeletePolicy) toProtoSoftDeletePolicy() *storagepb.Bucket_SoftDeletePolicy {
2103+
if p == nil {
2104+
return nil
2105+
}
2106+
// Excluding read only field EffectiveTime.
2107+
return &storagepb.Bucket_SoftDeletePolicy{
2108+
RetentionDuration: durationpb.New(p.RetentionDuration),
2109+
}
2110+
}
2111+
2112+
func toSoftDeletePolicyFromRaw(p *raw.BucketSoftDeletePolicy) *SoftDeletePolicy {
2113+
if p == nil {
2114+
return nil
2115+
}
2116+
2117+
policy := &SoftDeletePolicy{
2118+
RetentionDuration: time.Duration(p.RetentionDurationSeconds) * time.Second,
2119+
}
2120+
2121+
// Return EffectiveTime only if parsed to a valid value.
2122+
if t, err := time.Parse(time.RFC3339, p.EffectiveTime); err == nil {
2123+
policy.EffectiveTime = t
2124+
}
2125+
2126+
return policy
2127+
}
2128+
2129+
func toSoftDeletePolicyFromProto(p *storagepb.Bucket_SoftDeletePolicy) *SoftDeletePolicy {
2130+
if p == nil {
2131+
return nil
2132+
}
2133+
return &SoftDeletePolicy{
2134+
EffectiveTime: p.GetEffectiveTime().AsTime(),
2135+
RetentionDuration: p.GetRetentionDuration().AsDuration(),
2136+
}
2137+
}
2138+
20562139
// Objects returns an iterator over the objects in the bucket that match the
20572140
// Query q. If q is nil, no filtering is done. Objects will be iterated over
20582141
// lexicographically by name.

‎storage/bucket_test.go‎

Lines changed: 53 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,11 @@ func TestBucketAttrsToRawBucket(t *testing.T) {
6262
ResponseHeaders: []string{"FOO"},
6363
},
6464
},
65-
Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
66-
Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
67-
Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
68-
Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
65+
Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
66+
Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
67+
Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
68+
Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
69+
SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour},
6970
Lifecycle: Lifecycle{
7071
Rules: []LifecycleRule{{
7172
Action: LifecycleAction{
@@ -166,10 +167,11 @@ func TestBucketAttrsToRawBucket(t *testing.T) {
166167
ResponseHeader: []string{"FOO"},
167168
},
168169
},
169-
Encryption: &raw.BucketEncryption{DefaultKmsKeyName: "key"},
170-
Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
171-
Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
172-
Autoclass: &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
170+
Encryption: &raw.BucketEncryption{DefaultKmsKeyName: "key"},
171+
Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
172+
Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
173+
Autoclass: &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
174+
SoftDeletePolicy: &raw.BucketSoftDeletePolicy{RetentionDurationSeconds: 60 * 60},
173175
Lifecycle: &raw.BucketLifecycle{
174176
Rule: []*raw.BucketLifecycleRule{{
175177
Action: &raw.BucketLifecycleRuleAction{
@@ -395,10 +397,11 @@ func TestBucketAttrsToUpdateToRawBucket(t *testing.T) {
395397
},
396398
},
397399
},
398-
Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
399-
Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
400-
StorageClass: "NEARLINE",
401-
Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
400+
Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
401+
Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
402+
StorageClass: "NEARLINE",
403+
Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
404+
SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour},
402405
}
403406
au.SetLabel("a", "foo")
404407
au.DeleteLabel("b")
@@ -439,11 +442,12 @@ func TestBucketAttrsToUpdateToRawBucket(t *testing.T) {
439442
},
440443
},
441444
},
442-
Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
443-
Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
444-
StorageClass: "NEARLINE",
445-
Autoclass: &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "ARCHIVE", ForceSendFields: []string{"Enabled"}},
446-
ForceSendFields: []string{"DefaultEventBasedHold", "Lifecycle", "Autoclass"},
445+
Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
446+
Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
447+
StorageClass: "NEARLINE",
448+
Autoclass: &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "ARCHIVE", ForceSendFields: []string{"Enabled"}},
449+
SoftDeletePolicy: &raw.BucketSoftDeletePolicy{RetentionDurationSeconds: 3600},
450+
ForceSendFields: []string{"DefaultEventBasedHold", "Lifecycle", "Autoclass"},
447451
}
448452
if msg := testutil.Diff(got, want); msg != "" {
449453
t.Error(msg)
@@ -463,14 +467,15 @@ func TestBucketAttrsToUpdateToRawBucket(t *testing.T) {
463467

464468
// Test nulls.
465469
au3 := &BucketAttrsToUpdate{
466-
RetentionPolicy: &RetentionPolicy{},
467-
Encryption: &BucketEncryption{},
468-
Logging: &BucketLogging{},
469-
Website: &BucketWebsite{},
470+
RetentionPolicy: &RetentionPolicy{},
471+
Encryption: &BucketEncryption{},
472+
Logging: &BucketLogging{},
473+
Website: &BucketWebsite{},
474+
SoftDeletePolicy: &SoftDeletePolicy{},
470475
}
471476
got = au3.toRawBucket()
472477
want = &raw.Bucket{
473-
NullFields: []string{"RetentionPolicy", "Encryption", "Logging", "Website"},
478+
NullFields: []string{"RetentionPolicy", "Encryption", "Logging", "Website", "SoftDeletePolicy"},
474479
}
475480
if msg := testutil.Diff(got, want); msg != "" {
476481
t.Error(msg)
@@ -656,6 +661,10 @@ func TestNewBucket(t *testing.T) {
656661
TerminalStorageClass: "NEARLINE",
657662
TerminalStorageClassUpdateTime: "2017-10-23T04:05:06Z",
658663
},
664+
SoftDeletePolicy: &raw.BucketSoftDeletePolicy{
665+
EffectiveTime: "2017-10-23T04:05:06Z",
666+
RetentionDurationSeconds: 3600,
667+
},
659668
}
660669
want := &BucketAttrs{
661670
Name: "name",
@@ -713,6 +722,10 @@ func TestNewBucket(t *testing.T) {
713722
TerminalStorageClass: "NEARLINE",
714723
TerminalStorageClassUpdateTime: time.Date(2017, 10, 23, 4, 5, 6, 0, time.UTC),
715724
},
725+
SoftDeletePolicy: &SoftDeletePolicy{
726+
EffectiveTime: time.Date(2017, 10, 23, 4, 5, 6, 0, time.UTC),
727+
RetentionDuration: time.Hour,
728+
},
716729
}
717730
got, err := newBucket(rb)
718731
if err != nil {
@@ -768,6 +781,10 @@ func TestNewBucketFromProto(t *testing.T) {
768781
TerminalStorageClass: &autoclassTSC,
769782
TerminalStorageClassUpdateTime: toProtoTimestamp(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)),
770783
},
784+
SoftDeletePolicy: &storagepb.Bucket_SoftDeletePolicy{
785+
RetentionDuration: durationpb.New(3 * time.Hour),
786+
EffectiveTime: toProtoTimestamp(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)),
787+
},
771788
Lifecycle: &storagepb.Bucket_Lifecycle{
772789
Rule: []*storagepb.Bucket_Lifecycle_Rule{
773790
{
@@ -809,6 +826,10 @@ func TestNewBucketFromProto(t *testing.T) {
809826
Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
810827
Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
811828
Autoclass: &Autoclass{Enabled: true, ToggleTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), TerminalStorageClass: "NEARLINE", TerminalStorageClassUpdateTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)},
829+
SoftDeletePolicy: &SoftDeletePolicy{
830+
EffectiveTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),
831+
RetentionDuration: time.Hour * 3,
832+
},
812833
Lifecycle: Lifecycle{
813834
Rules: []LifecycleRule{{
814835
Action: LifecycleAction{
@@ -853,10 +874,11 @@ func TestBucketAttrsToProtoBucket(t *testing.T) {
853874
ResponseHeaders: []string{"FOO"},
854875
},
855876
},
856-
Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
857-
Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
858-
Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
859-
Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
877+
Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
878+
Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
879+
Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
880+
Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
881+
SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour * 2},
860882
Lifecycle: Lifecycle{
861883
Rules: []LifecycleRule{{
862884
Action: LifecycleAction{
@@ -903,10 +925,11 @@ func TestBucketAttrsToProtoBucket(t *testing.T) {
903925
ResponseHeader: []string{"FOO"},
904926
},
905927
},
906-
Encryption: &storagepb.Bucket_Encryption{DefaultKmsKey: "key"},
907-
Logging: &storagepb.Bucket_Logging{LogBucket: "projects/_/buckets/lb", LogObjectPrefix: "p"},
908-
Website: &storagepb.Bucket_Website{MainPageSuffix: "mps", NotFoundPage: "404"},
909-
Autoclass: &storagepb.Bucket_Autoclass{Enabled: true, TerminalStorageClass: &autoclassTSC},
928+
Encryption: &storagepb.Bucket_Encryption{DefaultKmsKey: "key"},
929+
Logging: &storagepb.Bucket_Logging{LogBucket: "projects/_/buckets/lb", LogObjectPrefix: "p"},
930+
Website: &storagepb.Bucket_Website{MainPageSuffix: "mps", NotFoundPage: "404"},
931+
Autoclass: &storagepb.Bucket_Autoclass{Enabled: true, TerminalStorageClass: &autoclassTSC},
932+
SoftDeletePolicy: &storagepb.Bucket_SoftDeletePolicy{RetentionDuration: durationpb.New(2 * time.Hour)},
910933
Lifecycle: &storagepb.Bucket_Lifecycle{
911934
Rule: []*storagepb.Bucket_Lifecycle_Rule{
912935
{

‎storage/client.go‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,9 @@ type storageClient interface {
5959
// Object metadata methods.
6060

6161
DeleteObject(ctx context.Context, bucket, object string, gen int64, conds *Conditions, opts ...storageOption) error
62-
GetObject(ctx context.Context, bucket, object string, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error)
62+
GetObject(ctx context.Context, params *getObjectParams, opts ...storageOption) (*ObjectAttrs, error)
6363
UpdateObject(ctx context.Context, params *updateObjectParams, opts ...storageOption) (*ObjectAttrs, error)
64+
RestoreObject(ctx context.Context, params *restoreObjectParams, opts ...storageOption) (*ObjectAttrs, error)
6465

6566
// Default Object ACL methods.
6667

@@ -294,6 +295,14 @@ type newRangeReaderParams struct {
294295
readCompressed bool // Use accept-encoding: gzip. Only works for HTTP currently.
295296
}
296297

298+
type getObjectParams struct {
299+
bucket, object string
300+
gen int64
301+
encryptionKey []byte
302+
conds *Conditions
303+
softDeleted bool
304+
}
305+
297306
type updateObjectParams struct {
298307
bucket, object string
299308
uattrs *ObjectAttrsToUpdate
@@ -303,6 +312,14 @@ type updateObjectParams struct {
303312
overrideRetention *bool
304313
}
305314

315+
type restoreObjectParams struct {
316+
bucket, object string
317+
gen int64
318+
encryptionKey []byte
319+
conds *Conditions
320+
copySourceACL bool
321+
}
322+
306323
type composeObjectRequest struct {
307324
dstBucket string
308325
dstObject destinationObject

‎storage/client_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,7 @@ func TestGetObjectEmulated(t *testing.T) {
272272
if err := w.Close(); err != nil {
273273
t.Fatalf("closing object: %v", err)
274274
}
275-
got, err := client.GetObject(context.Background(), bucket, want.Name, defaultGen, nil, nil)
275+
got, err := client.GetObject(context.Background(), &getObjectParams{bucket: bucket, object: want.Name, gen: defaultGen})
276276
if err != nil {
277277
t.Fatal(err)
278278
}
@@ -1332,7 +1332,7 @@ func TestObjectConditionsEmulated(t *testing.T) {
13321332
if err != nil {
13331333
return fmt.Errorf("creating object: %w", err)
13341334
}
1335-
_, err = client.GetObject(ctx, bucket, objName, gen, nil, &Conditions{GenerationMatch: gen, MetagenerationMatch: metaGen})
1335+
_, err = client.GetObject(ctx, &getObjectParams{bucket: bucket, object: objName, gen: gen, conds: &Conditions{GenerationMatch: gen, MetagenerationMatch: metaGen}})
13361336
return err
13371337
},
13381338
},

0 commit comments

Comments
 (0)