From 989f36fbb206832a6a3584c77546d3d560ac0df8 Mon Sep 17 00:00:00 2001 From: JesseLovelace <43148100+JesseLovelace@users.noreply.github.com> Date: Thu, 14 Mar 2024 15:40:32 -0700 Subject: [PATCH] feat: add soft delete feature (#2403) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add soft delete feature * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * add softDeleteTime and hardDeleteTime object fields * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * add new fields to field tests * clirr ignore * remove debug comments * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * add softdeletetime and harddeletetime to grpc codec * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix read mask test * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix style issues * updates to apiary library * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix read mask test * fix typo --------- Co-authored-by: Owl Bot --- .../clirr-ignored-differences.xml | 18 +++ .../java/com/google/cloud/storage/Blob.java | 12 ++ .../com/google/cloud/storage/BlobInfo.java | 50 +++++++- .../java/com/google/cloud/storage/Bucket.java | 28 +++++ .../com/google/cloud/storage/BucketInfo.java | 108 ++++++++++++++++++ .../google/cloud/storage/GrpcConversions.java | 33 ++++++ .../storage/GrpcRetryAlgorithmManager.java | 5 + .../google/cloud/storage/GrpcStorageImpl.java | 28 +++++ .../storage/HttpRetryAlgorithmManager.java | 5 + .../google/cloud/storage/JsonConversions.java | 31 +++++ .../com/google/cloud/storage/Storage.java | 101 +++++++++++++++- .../com/google/cloud/storage/StorageImpl.java | 18 +++ .../com/google/cloud/storage/UnifiedOpts.java | 71 ++++++++++++ .../cloud/storage/spi/v1/HttpStorageRpc.java | 34 +++++- .../storage/spi/v1/HttpStorageRpcSpans.java | 1 + .../cloud/storage/spi/v1/StorageRpc.java | 11 +- .../storage/testing/StorageRpcTestBase.java | 5 + .../cloud/storage/it/ITBlobReadMaskTest.java | 4 +- .../storage/it/ITBucketReadMaskTest.java | 24 +++- .../google/cloud/storage/it/ITBucketTest.java | 61 ++++++++++ .../storage/it/ITOptionRegressionTest.java | 12 +- .../runner/registry/AbstractStorageProxy.java | 5 + 22 files changed, 654 insertions(+), 11 deletions(-) diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml index 635e882a6..7fffdb6bd 100644 --- a/google-cloud-storage/clirr-ignored-differences.xml +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -27,6 +27,24 @@ com.google.cloud.storage.BlobInfo$Builder setRetention(com.google.cloud.storage.BlobInfo$Retention) + + 7013 + com/google/cloud/storage/BucketInfo$Builder + com.google.cloud.storage.BucketInfo$Builder setSoftDeletePolicy(com.google.cloud.storage.BucketInfo$SoftDeletePolicy) + + + + 7012 + com/google/cloud/storage/Storage + com.google.cloud.storage.Blob restore(com.google.cloud.storage.BlobId, com.google.cloud.storage.Storage$BlobRestoreOption[]) + + + + 7012 + com/google/cloud/storage/spi/v1/StorageRpc + com.google.api.services.storage.model.StorageObject restore(com.google.api.services.storage.model.StorageObject, java.util.Map) + + 7009 diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java index e6295b8c6..c6ea7a2ce 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java @@ -526,6 +526,18 @@ Builder setRetentionExpirationTimeOffsetDateTime(OffsetDateTime retentionExpirat return this; } + @Override + Builder setSoftDeleteTime(OffsetDateTime softDeleteTime) { + infoBuilder.setSoftDeleteTime(softDeleteTime); + return this; + } + + @Override + Builder setHardDeleteTime(OffsetDateTime hardDeleteTime) { + infoBuilder.setHardDeleteTime(hardDeleteTime); + return this; + } + @Override public Builder setRetention(Retention retention) { infoBuilder.setRetention(retention); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java index e808f444c..52ce09fd7 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java @@ -108,6 +108,8 @@ public class BlobInfo implements Serializable { private final Boolean temporaryHold; private final OffsetDateTime retentionExpirationTime; private final Retention retention; + private final OffsetDateTime softDeleteTime; + private final OffsetDateTime hardDeleteTime; private final transient ImmutableSet modifiedFields; /** This class is meant for internal use only. Users are discouraged from using this class. */ @@ -525,6 +527,10 @@ Builder setRetentionExpirationTimeOffsetDateTime(OffsetDateTime retentionExpirat return setRetentionExpirationTime(millisOffsetDateTimeCodec.decode(retentionExpirationTime)); } + abstract Builder setSoftDeleteTime(OffsetDateTime offsetDateTime); + + abstract Builder setHardDeleteTime(OffsetDateTime hardDeleteTIme); + public abstract Builder setRetention(Retention retention); /** Creates a {@code BlobInfo} object. */ @@ -626,6 +632,8 @@ static final class BuilderImpl extends Builder { private Boolean temporaryHold; private OffsetDateTime retentionExpirationTime; private Retention retention; + private OffsetDateTime softDeleteTime; + private OffsetDateTime hardDeleteTime; private final ImmutableSet.Builder modifiedFields = ImmutableSet.builder(); BuilderImpl(BlobId blobId) { @@ -664,6 +672,8 @@ static final class BuilderImpl extends Builder { temporaryHold = blobInfo.temporaryHold; retentionExpirationTime = blobInfo.retentionExpirationTime; retention = blobInfo.retention; + softDeleteTime = blobInfo.softDeleteTime; + hardDeleteTime = blobInfo.hardDeleteTime; } @Override @@ -1037,6 +1047,24 @@ Builder setRetentionExpirationTimeOffsetDateTime(OffsetDateTime retentionExpirat return this; } + @Override + Builder setSoftDeleteTime(OffsetDateTime softDeleteTime) { + if (!Objects.equals(this.softDeleteTime, softDeleteTime)) { + modifiedFields.add(BlobField.SOFT_DELETE_TIME); + } + this.softDeleteTime = softDeleteTime; + return this; + } + + @Override + Builder setHardDeleteTime(OffsetDateTime hardDeleteTime) { + if (!Objects.equals(this.hardDeleteTime, hardDeleteTime)) { + modifiedFields.add(BlobField.HARD_DELETE_TIME); + } + this.hardDeleteTime = hardDeleteTime; + return this; + } + @Override public Builder setRetention(Retention retention) { // todo: b/308194853 @@ -1269,6 +1297,8 @@ Builder clearRetentionExpirationTime() { temporaryHold = builder.temporaryHold; retentionExpirationTime = builder.retentionExpirationTime; retention = builder.retention; + softDeleteTime = builder.softDeleteTime; + hardDeleteTime = builder.hardDeleteTime; modifiedFields = builder.modifiedFields.build(); } @@ -1662,6 +1692,18 @@ public OffsetDateTime getRetentionExpirationTimeOffsetDateTime() { return retentionExpirationTime; } + /** If this object has been soft-deleted, returns the time it was soft-deleted. */ + public OffsetDateTime getSoftDeleteTime() { + return softDeleteTime; + } + + /** + * If this object has been soft-deleted, returns the time at which it will be permanently deleted. + */ + public OffsetDateTime getHardDeleteTime() { + return hardDeleteTime; + } + /** Returns the object's Retention policy. */ public Retention getRetention() { return retention; @@ -1717,7 +1759,9 @@ public int hashCode() { eventBasedHold, temporaryHold, retention, - retentionExpirationTime); + retentionExpirationTime, + softDeleteTime, + hardDeleteTime); } @Override @@ -1759,7 +1803,9 @@ public boolean equals(Object o) { && Objects.equals(eventBasedHold, blobInfo.eventBasedHold) && Objects.equals(temporaryHold, blobInfo.temporaryHold) && Objects.equals(retentionExpirationTime, blobInfo.retentionExpirationTime) - && Objects.equals(retention, blobInfo.retention); + && Objects.equals(retention, blobInfo.retention) + && Objects.equals(softDeleteTime, blobInfo.softDeleteTime) + && Objects.equals(hardDeleteTime, blobInfo.hardDeleteTime); } ImmutableSet getModifiedFields() { diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java index 6459e0a0a..0587bb0b2 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java @@ -748,6 +748,12 @@ Builder setObjectRetention(ObjectRetention objectRetention) { return this; } + @Override + public Builder setSoftDeletePolicy(SoftDeletePolicy softDeletePolicy) { + infoBuilder.setSoftDeletePolicy(softDeletePolicy); + return this; + } + @Override public Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace) { infoBuilder.setHierarchicalNamespace(hierarchicalNamespace); @@ -1089,6 +1095,28 @@ public Blob get(String blob, BlobGetOption... options) { return storage.get(BlobId.of(getName(), blob), options); } + /** + * Returns the requested blob in this bucket of a specific generation or {@code null} if not + * found. + * + *

Example of getting a blob of a specific in the bucket. + * + * 

{@code
+   * String blobName = "my_blob_name";
+   * long generation = 42;
+   * Blob blob = bucket.get(blobName, generation);
+   * }
+ * + * @param blob name of the requested blob + * @param generation the generation to get + * @param options blob search options + * @throws StorageException upon failure + */ + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + public Blob get(String blob, Long generation, BlobGetOption... options) { + return storage.get(BlobId.of(getName(), blob, generation), options); + } + /** * Returns a list of requested blobs in this bucket. Blobs that do not exist are null. * diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java index af3e4436e..323555922 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java @@ -120,6 +120,8 @@ public class BucketInfo implements Serializable { private final ObjectRetention objectRetention; private final HierarchicalNamespace hierarchicalNamespace; + private final SoftDeletePolicy softDeletePolicy; + private final transient ImmutableSet modifiedFields; /** @@ -350,6 +352,90 @@ public String toString() { } } + /** + * The bucket's soft delete policy. If this policy is set, any deleted objects will be + * soft-deleted according to the time specified in the policy + */ + public static class SoftDeletePolicy implements Serializable { + + private static final long serialVersionUID = -8100190443052242908L; + private Duration retentionDuration; + private OffsetDateTime effectiveTime; + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SoftDeletePolicy)) { + return false; + } + SoftDeletePolicy that = (SoftDeletePolicy) o; + return Objects.equals(retentionDuration, that.retentionDuration) + && Objects.equals(effectiveTime, that.effectiveTime); + } + + @Override + public int hashCode() { + return Objects.hash(retentionDuration, effectiveTime); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("retentionDuration", retentionDuration) + .add("effectiveTime", effectiveTime) + .toString(); + } + + public static Builder newBuilder() { + return new Builder(); + } + + public Builder toBuilder() { + return new Builder().setRetentionDuration(retentionDuration).setEffectiveTime(effectiveTime); + } + + private SoftDeletePolicy() {} + + public SoftDeletePolicy(Builder builder) { + this.retentionDuration = builder.retentionDuration; + this.effectiveTime = builder.effectiveTime; + } + + public Duration getRetentionDuration() { + return retentionDuration; + } + + public OffsetDateTime getEffectiveTime() { + return effectiveTime; + } + + public static final class Builder { + private Duration retentionDuration; + private OffsetDateTime effectiveTime; + + /** Sets the length of time to retain soft-deleted objects for, expressed as a Duration */ + public Builder setRetentionDuration(Duration retentionDuration) { + this.retentionDuration = retentionDuration; + return this; + } + + /** + * Sets the time from which this soft-delete policy is effective. This is package-private + * because it can only be set by the backend. + */ + Builder setEffectiveTime(OffsetDateTime effectiveTime) { + this.effectiveTime = effectiveTime; + return this; + } + + public SoftDeletePolicy build() { + return new SoftDeletePolicy(this); + } + } + } + /** * Configuration for the Autoclass settings of a bucket. * @@ -1753,6 +1839,8 @@ public Builder setRetentionPeriodDuration(Duration retentionPeriod) { abstract Builder setObjectRetention(ObjectRetention objectRetention); + public abstract Builder setSoftDeletePolicy(SoftDeletePolicy softDeletePolicy); + /** Creates a {@code BucketInfo} object. */ public abstract BucketInfo build(); @@ -1851,6 +1939,8 @@ static final class BuilderImpl extends Builder { private Logging logging; private CustomPlacementConfig customPlacementConfig; private ObjectRetention objectRetention; + + private SoftDeletePolicy softDeletePolicy; private HierarchicalNamespace hierarchicalNamespace; private final ImmutableSet.Builder modifiedFields = ImmutableSet.builder(); @@ -1891,6 +1981,7 @@ static final class BuilderImpl extends Builder { logging = bucketInfo.logging; customPlacementConfig = bucketInfo.customPlacementConfig; objectRetention = bucketInfo.objectRetention; + softDeletePolicy = bucketInfo.softDeletePolicy; hierarchicalNamespace = bucketInfo.hierarchicalNamespace; } @@ -2257,6 +2348,15 @@ Builder setObjectRetention(ObjectRetention objectRetention) { return this; } + @Override + public Builder setSoftDeletePolicy(SoftDeletePolicy softDeletePolicy) { + if (!Objects.equals(this.softDeletePolicy, softDeletePolicy)) { + modifiedFields.add(BucketField.SOFT_DELETE_POLICY); + } + this.softDeletePolicy = softDeletePolicy; + return this; + } + @Override public Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace) { if (!Objects.equals(this.hierarchicalNamespace, hierarchicalNamespace)) { @@ -2507,6 +2607,7 @@ private Builder clearDeleteLifecycleRules() { logging = builder.logging; customPlacementConfig = builder.customPlacementConfig; objectRetention = builder.objectRetention; + softDeletePolicy = builder.softDeletePolicy; hierarchicalNamespace = builder.hierarchicalNamespace; modifiedFields = builder.modifiedFields.build(); } @@ -2848,6 +2949,11 @@ public ObjectRetention getObjectRetention() { return objectRetention; } + /** returns the Soft Delete policy */ + public SoftDeletePolicy getSoftDeletePolicy() { + return softDeletePolicy; + } + /** Returns the Hierarchical Namespace (Folders) Configuration */ public HierarchicalNamespace getHierarchicalNamespace() { return hierarchicalNamespace; @@ -2890,6 +2996,7 @@ public int hashCode() { autoclass, locationType, objectRetention, + softDeletePolicy, hierarchicalNamespace, logging); } @@ -2932,6 +3039,7 @@ public boolean equals(Object o) { && Objects.equals(autoclass, that.autoclass) && Objects.equals(locationType, that.locationType) && Objects.equals(objectRetention, that.objectRetention) + && Objects.equals(softDeletePolicy, that.softDeletePolicy) && Objects.equals(hierarchicalNamespace, that.hierarchicalNamespace) && Objects.equals(logging, that.logging); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java index d1084bc41..28c51873b 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java @@ -16,6 +16,7 @@ package com.google.cloud.storage; +import static com.google.cloud.storage.Storage.BucketField.SOFT_DELETE_POLICY; import static com.google.cloud.storage.Utils.bucketNameCodec; import static com.google.cloud.storage.Utils.ifNonNull; import static com.google.cloud.storage.Utils.lift; @@ -90,6 +91,9 @@ final class GrpcConversions { Codec.of(this::iamConfigEncode, this::iamConfigDecode); private final Codec autoclassCodec = Codec.of(this::autoclassEncode, this::autoclassDecode); + + private final Codec softDeletePolicyCodec = + Codec.of(this::softDeletePolicyEncode, this::softDeletePolicyDecode); private final Codec lifecycleRuleCodec = Codec.of(this::lifecycleRuleEncode, this::lifecycleRuleDecode); private final Codec bucketInfoCodec = @@ -294,6 +298,9 @@ private BucketInfo bucketInfoDecode(Bucket from) { if (from.hasAutoclass()) { to.setAutoclass(autoclassCodec.decode(from.getAutoclass())); } + if (from.hasSoftDeletePolicy()) { + to.setSoftDeletePolicy(softDeletePolicyCodec.decode(from.getSoftDeletePolicy())); + } if (from.hasCustomPlacementConfig()) { Bucket.CustomPlacementConfig customPlacementConfig = from.getCustomPlacementConfig(); to.setCustomPlacementConfig( @@ -383,6 +390,11 @@ private Bucket bucketInfoEncode(BucketInfo from) { ifNonNull(from.getAcl(), toImmutableListOf(bucketAclCodec::encode), to::addAllAcl); ifNonNull(from.getIamConfiguration(), iamConfigurationCodec::encode, to::setIamConfig); ifNonNull(from.getAutoclass(), autoclassCodec::encode, to::setAutoclass); + ifNonNull(from.getSoftDeletePolicy(), softDeletePolicyCodec::encode, to::setSoftDeletePolicy); + if (from.getModifiedFields().contains(SOFT_DELETE_POLICY) + && from.getSoftDeletePolicy() == null) { + to.clearSoftDeletePolicy(); + } CustomPlacementConfig customPlacementConfig = from.getCustomPlacementConfig(); if (customPlacementConfig != null && customPlacementConfig.getDataLocations() != null) { to.setCustomPlacementConfig( @@ -601,6 +613,19 @@ private Bucket.Autoclass autoclassEncode(BucketInfo.Autoclass from) { return to.build(); } + private BucketInfo.SoftDeletePolicy softDeletePolicyDecode(Bucket.SoftDeletePolicy from) { + BucketInfo.SoftDeletePolicy.Builder to = BucketInfo.SoftDeletePolicy.newBuilder(); + ifNonNull(from.getRetentionDuration(), durationCodec::decode, to::setRetentionDuration); + ifNonNull(from.getEffectiveTime(), timestampCodec::decode, to::setEffectiveTime); + return to.build(); + } + + private Bucket.SoftDeletePolicy softDeletePolicyEncode(BucketInfo.SoftDeletePolicy from) { + Bucket.SoftDeletePolicy.Builder to = Bucket.SoftDeletePolicy.newBuilder(); + ifNonNull(from.getRetentionDuration(), durationCodec::encode, to::setRetentionDuration); + return to.build(); + } + private Bucket.HierarchicalNamespace hierarchicalNamespaceEncode( BucketInfo.HierarchicalNamespace from) { Bucket.HierarchicalNamespace.Builder to = Bucket.HierarchicalNamespace.newBuilder(); @@ -863,6 +888,8 @@ private Object blobInfoEncode(BlobInfo from) { ifNonNull(from.getUpdateTimeOffsetDateTime(), timestampCodec::encode, toBuilder::setUpdateTime); ifNonNull(from.getCreateTimeOffsetDateTime(), timestampCodec::encode, toBuilder::setCreateTime); ifNonNull(from.getCustomTimeOffsetDateTime(), timestampCodec::encode, toBuilder::setCustomTime); + ifNonNull(from.getSoftDeleteTime(), timestampCodec::encode, toBuilder::setSoftDeleteTime); + ifNonNull(from.getHardDeleteTime(), timestampCodec::encode, toBuilder::setHardDeleteTime); ifNonNull( from.getCustomerEncryption(), customerEncryptionCodec::encode, @@ -928,6 +955,12 @@ private BlobInfo blobInfoDecode(Object from) { if (from.hasCustomerEncryption()) { toBuilder.setCustomerEncryption(customerEncryptionCodec.decode(from.getCustomerEncryption())); } + if (from.hasSoftDeleteTime()) { + toBuilder.setSoftDeleteTime(timestampCodec.decode(from.getSoftDeleteTime())); + } + if (from.hasHardDeleteTime()) { + toBuilder.setHardDeleteTime(timestampCodec.decode(from.getHardDeleteTime())); + } String storageClass = from.getStorageClass(); if (!storageClass.isEmpty()) { toBuilder.setStorageClass(StorageClass.valueOf(storageClass)); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcRetryAlgorithmManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcRetryAlgorithmManager.java index ea481ab25..de7af195b 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcRetryAlgorithmManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcRetryAlgorithmManager.java @@ -41,6 +41,7 @@ import com.google.storage.v2.LockBucketRetentionPolicyRequest; import com.google.storage.v2.QueryWriteStatusRequest; import com.google.storage.v2.ReadObjectRequest; +import com.google.storage.v2.RestoreObjectRequest; import com.google.storage.v2.RewriteObjectRequest; import com.google.storage.v2.StartResumableWriteRequest; import com.google.storage.v2.UpdateBucketRequest; @@ -123,6 +124,10 @@ public ResultRetryAlgorithm getFor(GetObjectRequest req) { return retryStrategy.getIdempotentHandler(); } + public ResultRetryAlgorithm getFor(RestoreObjectRequest req) { + return retryStrategy.getIdempotentHandler(); + } + public ResultRetryAlgorithm getFor(GetServiceAccountRequest req) { return retryStrategy.getIdempotentHandler(); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java index 53c73d4d5..e9857e93d 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java @@ -112,6 +112,7 @@ import com.google.storage.v2.Object; import com.google.storage.v2.ObjectAccessControl; import com.google.storage.v2.ReadObjectRequest; +import com.google.storage.v2.RestoreObjectRequest; import com.google.storage.v2.RewriteObjectRequest; import com.google.storage.v2.RewriteResponse; import com.google.storage.v2.StorageClient; @@ -403,6 +404,33 @@ public Blob get(BlobId blob) { return get(blob, new BlobGetOption[0]); } + @Override + public Blob restore(BlobId blob, BlobRestoreOption... options) { + Opts unwrap = Opts.unwrap(options); + return internalObjectRestore(blob, unwrap); + } + + private Blob internalObjectRestore(BlobId blobId, Opts opts) { + Opts finalOpts = opts.prepend(defaultOpts).prepend(ALL_BLOB_FIELDS); + GrpcCallContext grpcCallContext = + finalOpts.grpcMetadataMapper().apply(GrpcCallContext.createDefault()); + RestoreObjectRequest.Builder builder = + RestoreObjectRequest.newBuilder() + .setBucket(bucketNameCodec.encode(blobId.getBucket())) + .setObject(blobId.getName()); + ifNonNull(blobId.getGeneration(), builder::setGeneration); + RestoreObjectRequest req = finalOpts.restoreObjectRequest().apply(builder).build(); + GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext()); + return Retrying.run( + getOptions(), + retryAlgorithmManager.getFor(req), + () -> storageClient.restoreObjectCallable().call(req, merge), + resp -> { + BlobInfo tmp = codecs.blobInfo().decode(resp); + return finalOpts.clearBlobFields().decode(tmp).asBlob(this); + }); + } + @Override public Page list(BucketListOption... options) { Opts opts = Opts.unwrap(options).prepend(defaultOpts).prepend(ALL_BUCKET_FIELDS); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRetryAlgorithmManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRetryAlgorithmManager.java index 22cac9a70..c5163ad00 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRetryAlgorithmManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRetryAlgorithmManager.java @@ -213,6 +213,11 @@ public ResultRetryAlgorithm getForObjectsGet( return retryStrategy.getIdempotentHandler(); } + public ResultRetryAlgorithm getForObjectsRestore( + StorageObject pb, Map optionsMap) { + return retryStrategy.getIdempotentHandler(); + } + public ResultRetryAlgorithm getForObjectsUpdate( StorageObject pb, Map optionsMap) { return optionsMap.containsKey(StorageRpc.Option.IF_METAGENERATION_MATCH) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java index c5add241d..7f84db77d 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java @@ -16,6 +16,7 @@ package com.google.cloud.storage; +import static com.google.cloud.storage.Storage.BucketField.SOFT_DELETE_POLICY; import static com.google.cloud.storage.Utils.dateTimeCodec; import static com.google.cloud.storage.Utils.durationSecondsCodec; import static com.google.cloud.storage.Utils.ifNonNull; @@ -66,6 +67,7 @@ import com.google.cloud.storage.BucketInfo.Logging; import com.google.cloud.storage.BucketInfo.ObjectRetention; import com.google.cloud.storage.BucketInfo.PublicAccessPrevention; +import com.google.cloud.storage.BucketInfo.SoftDeletePolicy; import com.google.cloud.storage.Conversions.Codec; import com.google.cloud.storage.Cors.Origin; import com.google.cloud.storage.HmacKey.HmacKeyMetadata; @@ -119,6 +121,9 @@ final class JsonConversions { private final Codec objectRetentionCodec = Codec.of(this::objectRetentionEncode, this::objectRetentionDecode); + + private final Codec softDeletePolicyCodec = + Codec.of(this::softDeletePolicyEncode, this::softDeletePolicyDecode); private final Codec lifecycleRuleCodec = Codec.of(this::lifecycleRuleEncode, this::lifecycleRuleDecode); private final Codec lifecycleConditionCodec = @@ -251,6 +256,9 @@ private StorageObject blobInfoEncode(BlobInfo from) { dateTimeCodec::encode, to::setRetentionExpirationTime); + ifNonNull(from.getSoftDeleteTime(), dateTimeCodec::encode, to::setSoftDeleteTime); + ifNonNull(from.getHardDeleteTime(), dateTimeCodec::encode, to::setHardDeleteTime); + // todo: clean this up once retention is enabled in grpc // This is a workaround so that explicitly null retention objects are only included when the // user set an existing policy to null, to avoid sending any retention objects to the test @@ -332,6 +340,8 @@ private BlobInfo blobInfoDecode(StorageObject from) { dateTimeCodec::decode, to::setRetentionExpirationTimeOffsetDateTime); ifNonNull(from.getRetention(), this::retentionDecode, to::setRetention); + ifNonNull(from.getSoftDeleteTime(), dateTimeCodec::decode, to::setSoftDeleteTime); + ifNonNull(from.getHardDeleteTime(), dateTimeCodec::decode, to::setHardDeleteTime); return to.build(); } @@ -371,6 +381,21 @@ private Retention retentionDecode(StorageObject.Retention from) { return to.build(); } + private Bucket.SoftDeletePolicy softDeletePolicyEncode(SoftDeletePolicy from) { + Bucket.SoftDeletePolicy to = new Bucket.SoftDeletePolicy(); + ifNonNull( + from.getRetentionDuration(), durationSecondsCodec::encode, to::setRetentionDurationSeconds); + return to; + } + + private SoftDeletePolicy softDeletePolicyDecode(Bucket.SoftDeletePolicy from) { + SoftDeletePolicy.Builder to = SoftDeletePolicy.newBuilder(); + ifNonNull( + from.getRetentionDurationSeconds(), durationSecondsCodec::decode, to::setRetentionDuration); + ifNonNull(from.getEffectiveTime(), dateTimeCodec::decode, to::setEffectiveTime); + return to.build(); + } + private Bucket bucketInfoEncode(BucketInfo from) { Bucket to = new Bucket(); ifNonNull(from.getProject(), projectNameCodec::encode, p -> to.set(PROJECT_ID_FIELD_NAME, p)); @@ -441,6 +466,11 @@ private Bucket bucketInfoEncode(BucketInfo from) { this::customPlacementConfigEncode, to::setCustomPlacementConfig); ifNonNull(from.getObjectRetention(), this::objectRetentionEncode, to::setObjectRetention); + ifNonNull(from.getSoftDeletePolicy(), this::softDeletePolicyEncode, to::setSoftDeletePolicy); + if (from.getSoftDeletePolicy() == null + && from.getModifiedFields().contains(SOFT_DELETE_POLICY)) { + to.setSoftDeletePolicy(Data.nullOf(Bucket.SoftDeletePolicy.class)); + } ifNonNull( from.getHierarchicalNamespace(), this::hierarchicalNamespaceEncode, @@ -500,6 +530,7 @@ private BucketInfo bucketInfoDecode(com.google.api.services.storage.model.Bucket this::hierarchicalNamespaceDecode, to::setHierarchicalNamespace); ifNonNull(from.getObjectRetention(), this::objectRetentionDecode, to::setObjectRetention); + ifNonNull(from.getSoftDeletePolicy(), this::softDeletePolicyDecode, to::setSoftDeletePolicy); return to.build(); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 1d0ae8347..0f6b002d9 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -163,7 +163,10 @@ enum BucketField implements FieldSelector, NamedField { @TransportCompatibility({Transport.HTTP, Transport.GRPC}) HIERARCHICAL_NAMESPACE("hierarchicalNamespace", "hierarchical_namespace"), @TransportCompatibility({Transport.HTTP}) - OBJECT_RETENTION("objectRetention"); + OBJECT_RETENTION("objectRetention"), + + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + SOFT_DELETE_POLICY("softDeletePolicy", "soft_delete_policy"); static final List REQUIRED_FIELDS = ImmutableList.of(NAME); @@ -263,7 +266,13 @@ enum BlobField implements FieldSelector, NamedField { @TransportCompatibility({Transport.HTTP, Transport.GRPC}) CUSTOMER_ENCRYPTION("customerEncryption", "customer_encryption"), @TransportCompatibility({Transport.HTTP}) - RETENTION("retention"); + RETENTION("retention"), + + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + SOFT_DELETE_TIME("softDeleteTime", "soft_delete_time"), + + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + HARD_DELETE_TIME("hardDeleteTime", "hard_delete_time"); static final List REQUIRED_FIELDS = ImmutableList.of(BUCKET, NAME); @@ -1572,6 +1581,16 @@ public static BlobGetOption shouldReturnRawInputStream(boolean shouldReturnRawIn return new BlobGetOption(UnifiedOpts.returnRawInputStream(shouldReturnRawInputStream)); } + /** + * Returns an option for whether the request should return a soft-deleted object. If an object + * has been soft-deleted (Deleted while a Soft Delete Policy) is in place, this must be true or + * the request will return null. + */ + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + public static BlobGetOption softDeleted(boolean softDeleted) { + return new BlobGetOption(UnifiedOpts.softDeleted(softDeleted)); + } + /** * Deduplicate any options which are the same parameter. The value which comes last in {@code * os} will be the value included in the return. @@ -1607,6 +1626,61 @@ public static BlobGetOption[] dedupe(BlobGetOption[] array, BlobGetOption... os) } } + /** Class for specifying blob restore options * */ + class BlobRestoreOption extends Option { + + private static final long serialVersionUID = 1922118465380110958L; + + BlobRestoreOption(ObjectSourceOpt opt) { + super(opt); + } + + /** + * Returns an option for blob's data generation match. If this option is used the request will + * fail if generation does not match. + */ + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + public static BlobRestoreOption generationMatch(long generation) { + return new BlobRestoreOption(UnifiedOpts.generationMatch(generation)); + } + + /** + * Returns an option for blob's data generation mismatch. If this option is used the request + * will fail if blob's generation matches the provided value. + */ + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + public static BlobRestoreOption generationNotMatch(long generation) { + return new BlobRestoreOption(UnifiedOpts.generationNotMatch(generation)); + } + + /** + * Returns an option for blob's metageneration match. If this option is used the request will + * fail if blob's metageneration does not match the provided value. + */ + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + public static BlobRestoreOption metagenerationMatch(long generation) { + return new BlobRestoreOption(UnifiedOpts.metagenerationMatch(generation)); + } + + /** + * Returns an option for blob's metageneration mismatch. If this option is used the request will + * fail if blob's metageneration matches the provided value. + */ + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + public static BlobRestoreOption metagenerationNotMatch(long generation) { + return new BlobRestoreOption(UnifiedOpts.metagenerationNotMatch(generation)); + } + + /** + * Returns an option for whether the restored object should copy the access controls of the + * source object. + */ + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + public static BlobRestoreOption copySourceAcl(boolean copySourceAcl) { + return new BlobRestoreOption(UnifiedOpts.copySourceAcl(copySourceAcl)); + } + } + /** Class for specifying bucket list options. */ class BucketListOption extends Option { @@ -1838,6 +1912,12 @@ public static BlobListOption fields(BlobField... fields) { return new BlobListOption(UnifiedOpts.fields(set)); } + /** Returns an option for whether the list result should include soft-deleted objects. */ + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + public static BlobListOption softDeleted(boolean softDeleted) { + return new BlobListOption(UnifiedOpts.softDeleted(softDeleted)); + } + /** * Deduplicate any options which are the same parameter. The value which comes last in {@code * os} will be the value included in the return. @@ -3043,6 +3123,23 @@ Blob createFrom( @TransportCompatibility({Transport.HTTP, Transport.GRPC}) Blob get(BlobId blob); + /** + * Restores a soft-deleted object to full object status and returns the object. Note that you must + * specify a generation to use this method. + * + *

Example of restoring an object. + * + * 

{@code
+   * String bucketName = "my-unique-bucket";
+   * String blobName = "my-blob-name";
+   * long generation = 42;
+   * BlobId blobId = BlobId.of(bucketName, blobName, gen);
+   * Blob blob = storage.restore(blobId);
+   * }
+ */ + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + Blob restore(BlobId blob, BlobRestoreOption... options); + /** * Lists the project's buckets. * diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 9af54fee2..a5df68b83 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -341,6 +341,24 @@ public Blob get(BlobId blob) { return get(blob, new BlobGetOption[0]); } + @Override + public Blob restore(BlobId blob, BlobRestoreOption... options) { + ImmutableMap optionsMap = + Opts.unwrap(options).resolveFrom(blob).getRpcOptions(); + + StorageObject obj = codecs.blobId().encode(blob); + + ResultRetryAlgorithm algorithm = retryAlgorithmManager.getForObjectsRestore(obj, optionsMap); + + return run( + algorithm, + () -> storageRpc.restore(obj, optionsMap), + (x) -> { + BlobInfo info = Conversions.json().blobInfo().decode(x); + return info.asBlob(this); + }); + } + private static class BucketPageFetcher implements NextPageFetcher { private static final long serialVersionUID = 8534413447247364038L; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java index 3159cbebb..06909620d 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java @@ -53,6 +53,7 @@ import com.google.storage.v2.ListObjectsRequest; import com.google.storage.v2.LockBucketRetentionPolicyRequest; import com.google.storage.v2.ReadObjectRequest; +import com.google.storage.v2.RestoreObjectRequest; import com.google.storage.v2.RewriteObjectRequest; import com.google.storage.v2.UpdateBucketRequest; import com.google.storage.v2.UpdateHmacKeyRequest; @@ -157,6 +158,10 @@ default Mapper getObject() { default Mapper rewriteObject() { return Mapper.identity(); } + + default Mapper restoreObject() { + return Mapper.identity(); + } } /** @@ -482,6 +487,14 @@ static Projection projection(@NonNull String projection) { return new Projection(projection); } + static SoftDeleted softDeleted(boolean softDeleted) { + return new SoftDeleted(softDeleted); + } + + static CopySourceAcl copySourceAcl(boolean copySourceAcl) { + return new CopySourceAcl(copySourceAcl); + } + static RequestedPolicyVersion requestedPolicyVersion(long l) { return new RequestedPolicyVersion(l); } @@ -667,6 +680,40 @@ public Mapper listObjects() { } } + static final class SoftDeleted extends RpcOptVal + implements ObjectListOpt, ObjectSourceOpt { + + private static final long serialVersionUID = -8526951678111463350L; + + private SoftDeleted(boolean val) { + super(StorageRpc.Option.SOFT_DELETED, val); + } + + @Override + public Mapper listObjects() { + return b -> b.setSoftDeleted(val); + } + + @Override + public Mapper getObject() { + return b -> b.setSoftDeleted(val); + } + } + + static final class CopySourceAcl extends RpcOptVal implements ObjectSourceOpt { + + private static final long serialVersionUID = 2033755749149128119L; + + private CopySourceAcl(boolean val) { + super(StorageRpc.Option.COPY_SOURCE_ACL, val); + } + + @Override + public Mapper restoreObject() { + return b -> b.setCopySourceAcl(val); + } + } + static final class DisableGzipContent extends RpcOptVal<@NonNull Boolean> implements ObjectTargetOpt { private static final long serialVersionUID = 7445066765944965549L; @@ -1008,6 +1055,11 @@ public Mapper getObject() { return b -> b.setIfGenerationMatch(val); } + @Override + public Mapper restoreObject() { + return b -> b.setIfGenerationMatch(val); + } + @Override public Mapper updateObject() { return b -> b.setIfGenerationMatch(val); @@ -1064,6 +1116,11 @@ public Mapper getObject() { return b -> b.setIfGenerationNotMatch(val); } + @Override + public Mapper restoreObject() { + return b -> b.setIfGenerationNotMatch(val); + } + @Override public Mapper updateObject() { return b -> b.setIfGenerationNotMatch(val); @@ -1205,6 +1262,11 @@ public Mapper getObject() { return b -> b.setIfMetagenerationMatch(val); } + @Override + public Mapper restoreObject() { + return b -> b.setIfMetagenerationMatch(val); + } + @Override public Mapper updateObject() { return b -> b.setIfMetagenerationMatch(val); @@ -1285,6 +1347,11 @@ public Mapper getObject() { return b -> b.setIfMetagenerationNotMatch(val); } + @Override + public Mapper restoreObject() { + return b -> b.setIfMetagenerationNotMatch(val); + } + @Override public Mapper updateObject() { return b -> b.setIfMetagenerationNotMatch(val); @@ -2297,6 +2364,10 @@ Mapper getObjectsRequest() { return fuseMappers(ObjectSourceOpt.class, ObjectSourceOpt::getObject); } + Mapper restoreObjectRequest() { + return fuseMappers(ObjectSourceOpt.class, ObjectSourceOpt::restoreObject); + } + Mapper readObjectRequest() { return fuseMappers(ObjectSourceOpt.class, ObjectSourceOpt::readObject); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 3ca2eabec..8a9734271 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -459,6 +459,7 @@ public Tuple> list(final String bucket, Map storageObjects = @@ -541,7 +542,8 @@ private Storage.Objects.Get getCall(StorageObject object, Map options .setIfGenerationMatch(Option.IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(Option.IF_GENERATION_NOT_MATCH.getLong(options)) .setFields(Option.FIELDS.getString(options)) - .setUserProject(Option.USER_PROJECT.getString(options)); + .setUserProject(Option.USER_PROJECT.getString(options)) + .setSoftDeleted(Option.SOFT_DELETED.getBoolean(options)); } @Override @@ -563,6 +565,36 @@ public StorageObject get(StorageObject object, Map options) { } } + @Override + public StorageObject restore(StorageObject object, Map options) { + Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_RESTORE_OBJECT); + Scope scope = tracer.withSpan(span); + try { + Storage.Objects.Restore restore = + storage.objects().restore(object.getBucket(), object.getName(), object.getGeneration()); + return restore + .setProjection(DEFAULT_PROJECTION) + .setIfMetagenerationMatch(Option.IF_METAGENERATION_MATCH.getLong(options)) + .setIfMetagenerationNotMatch(Option.IF_METAGENERATION_NOT_MATCH.getLong(options)) + .setIfGenerationMatch(Option.IF_GENERATION_MATCH.getLong(options)) + .setIfGenerationNotMatch(Option.IF_GENERATION_NOT_MATCH.getLong(options)) + .setCopySourceAcl(Option.COPY_SOURCE_ACL.getBoolean(options)) + .setUserProject(Option.USER_PROJECT.getString(options)) + .setFields(Option.FIELDS.getString(options)) + .execute(); + } catch (IOException ex) { + span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); + StorageException serviceException = translate(ex); + if (serviceException.getCode() == HTTP_NOT_FOUND) { + return null; + } + throw serviceException; + } finally { + scope.close(); + span.end(HttpStorageRpcSpans.END_SPAN_OPTIONS); + } + } + @Override public Bucket patch(Bucket bucket, Map options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_PATCH_BUCKET); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java index 3f3d27d94..dc4b05336 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java @@ -30,6 +30,7 @@ class HttpStorageRpcSpans { static final String SPAN_NAME_LIST_OBJECTS = getTraceSpanName("list(String,Map)"); static final String SPAN_NAME_GET_BUCKET = getTraceSpanName("get(Bucket,Map)"); static final String SPAN_NAME_GET_OBJECT = getTraceSpanName("get(StorageObject,Map)"); + static final String SPAN_NAME_RESTORE_OBJECT = getTraceSpanName("restore(StorageObject, Map)"); static final String SPAN_NAME_PATCH_BUCKET = getTraceSpanName("patch(Bucket,Map)"); static final String SPAN_NAME_PATCH_OBJECT = getTraceSpanName("patch(StorageObject,Map)"); static final String SPAN_NAME_DELETE_BUCKET = getTraceSpanName("delete(Bucket,Map)"); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java index 3b40f6a23..d4e0abbff 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java @@ -74,8 +74,10 @@ enum Option { ENABLE_OBJECT_RETENTION("enableObjectRetention"), RETURN_RAW_INPUT_STREAM("returnRawInputStream"), OVERRIDE_UNLOCKED_RETENTION("overrideUnlockedRetention"), + SOFT_DELETED("softDeleted"), + COPY_SOURCE_ACL("copySourceAcl"), + GENERATION("generation"), INCLUDE_FOLDERS_AS_PREFIXES("includeFoldersAsPrefixes"); - ; private final String value; @@ -243,6 +245,13 @@ public int hashCode() { */ StorageObject get(StorageObject object, Map options); + /** + * If an object has been soft-deleted, restores it and returns the restored object.j + * + * @throws StorageException upon failure + */ + StorageObject restore(StorageObject object, Map options); + /** * Updates bucket information. * diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java index 97104634f..6686cb925 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java @@ -71,6 +71,11 @@ public StorageObject get(StorageObject object, Map options) { throw new UnsupportedOperationException("Not implemented yet"); } + @Override + public StorageObject restore(StorageObject object, Map options) { + throw new UnsupportedOperationException("Not implemented yet"); + } + @Override public Bucket patch(Bucket bucket, Map options) { throw new UnsupportedOperationException("Not implemented yet"); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobReadMaskTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobReadMaskTest.java index c2594a484..bf8c48258 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobReadMaskTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobReadMaskTest.java @@ -201,7 +201,9 @@ public ImmutableList parameters() { new Args<>(BlobField.UPDATED, LazyAssertion.equal()), new Args<>( BlobField.RETENTION, - LazyAssertion.skip("TODO: jesse fill in buganizer bug here"))); + LazyAssertion.skip("TODO: jesse fill in buganizer bug here")), + new Args<>(BlobField.SOFT_DELETE_TIME, LazyAssertion.equal()), + new Args<>(BlobField.HARD_DELETE_TIME, LazyAssertion.equal())); List argsDefined = args.stream().map(Args::getField).map(Enum::name).sorted().collect(Collectors.toList()); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java index 9c9776032..c4c3c4059 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java @@ -40,6 +40,7 @@ import com.google.cloud.storage.it.runner.annotations.SingleBackend; import com.google.cloud.storage.it.runner.annotations.StorageFixture; import com.google.common.collect.ImmutableList; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -133,8 +134,27 @@ public ImmutableList parameters() { new Args<>(BucketField.TIME_CREATED, LazyAssertion.equal()), new Args<>(BucketField.UPDATED, LazyAssertion.equal()), new Args<>(BucketField.VERSIONING, LazyAssertion.equal()), - new Args<>(BucketField.HIERARCHICAL_NAMESPACE, LazyAssertion.equal()), - new Args<>(BucketField.WEBSITE, LazyAssertion.equal())); + new Args<>(BucketField.WEBSITE, LazyAssertion.equal()), + new Args<>( + BucketField.SOFT_DELETE_POLICY, + (jsonT, grpcT) -> { + assertThat( + jsonT + .getSoftDeletePolicy() + .getRetentionDuration() + .equals(grpcT.getSoftDeletePolicy().getRetentionDuration())); + assertThat( + jsonT + .getSoftDeletePolicy() + .getEffectiveTime() + .truncatedTo(ChronoUnit.SECONDS) + .equals( + grpcT + .getSoftDeletePolicy() + .getEffectiveTime() + .truncatedTo(ChronoUnit.SECONDS))); + }), + new Args<>(BucketField.HIERARCHICAL_NAMESPACE, LazyAssertion.equal())); List argsDefined = args.stream().map(Args::getField).map(Enum::name).sorted().collect(Collectors.toList()); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java index 5b759be6b..97ab8ad3e 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java @@ -554,6 +554,67 @@ public void testUpdateBucket_noModification() throws Exception { } } + @Test + public void testSoftDeletePolicy() { + String bucketName = generator.randomBucketName(); + BucketInfo bucketInfo = + BucketInfo.newBuilder(bucketName) + .setSoftDeletePolicy( + BucketInfo.SoftDeletePolicy.newBuilder() + .setRetentionDuration(Duration.ofDays(10)) + .build()) + .build(); + try { + storage.create(bucketInfo); + + Bucket remoteBucket = storage.get(bucketName); + assertEquals(Duration.ofDays(10), remoteBucket.getSoftDeletePolicy().getRetentionDuration()); + assertNotNull(remoteBucket.getSoftDeletePolicy().getEffectiveTime()); + + String softDelBlobName = "softdelblob"; + remoteBucket.create(softDelBlobName, BLOB_BYTE_CONTENT); + + Blob blob = remoteBucket.get(softDelBlobName); + long gen = blob.getGeneration(); + + assertNull(blob.getSoftDeleteTime()); + assertNull(blob.getHardDeleteTime()); + + blob.delete(); + + assertNull(remoteBucket.get(softDelBlobName)); + + ImmutableList softDeletedBlobs = + ImmutableList.copyOf( + remoteBucket.list(Storage.BlobListOption.softDeleted(true)).iterateAll()); + assertThat(softDeletedBlobs.size() > 0); + + Blob softDeletedBlob = + remoteBucket.get(softDelBlobName, gen, Storage.BlobGetOption.softDeleted(true)); + + assertNotNull(softDeletedBlob); + assertNotNull(softDeletedBlob.getSoftDeleteTime()); + assertNotNull(softDeletedBlob.getHardDeleteTime()); + + assertNotNull(storage.restore(softDeletedBlob.getBlobId())); + + remoteBucket + .toBuilder() + .setSoftDeletePolicy( + BucketInfo.SoftDeletePolicy.newBuilder() + .setRetentionDuration(Duration.ofDays(20)) + .build()) + .build() + .update(); + + assertEquals( + Duration.ofDays(20), + storage.get(bucketName).getSoftDeletePolicy().getRetentionDuration()); + } finally { + BucketCleaner.doCleanup(bucketName, storage); + } + } + @Test public void createBucketWithHierarchicalNamespace() { String bucketName = generator.randomBucketName(); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java index 7627bf3af..c7032ea61 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java @@ -326,6 +326,8 @@ public void storage_BucketGetOption_fields_BucketField() { "timeCreated", "updated", "versioning", + "website", + "softDeletePolicy", "hierarchicalNamespace", "website"); s.get( @@ -739,7 +741,9 @@ public void storage_BlobGetOption_fields_BlobField() { "timeDeleted", "timeStorageClassUpdated", "updated", - "retention"); + "retention", + "softDeleteTime", + "hardDeleteTime"); s.get(o.getBlobId(), BlobGetOption.fields(BlobField.values())); requestAuditing.assertQueryParam("fields", expected, splitOnCommaToSet()); } @@ -817,6 +821,8 @@ public void storage_BucketListOption_fields_BucketField() { "items/timeCreated", "items/updated", "items/versioning", + "items/website", + "items/softDeletePolicy", "items/hierarchicalNamespace", "items/website"); s.list(BucketListOption.fields(TestUtils.filterOutHttpOnlyBucketFields(BucketField.values()))); @@ -915,7 +921,9 @@ public void storage_BlobListOption_fields_BlobField() { "items/timeDeleted", "items/timeStorageClassUpdated", "items/updated", - "items/retention"); + "items/retention", + "items/softDeleteTime", + "items/hardDeleteTime"); s.list(b.getName(), BlobListOption.fields(BlobField.values())); requestAuditing.assertQueryParam("fields", expected, splitOnCommaToSet()); } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/AbstractStorageProxy.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/AbstractStorageProxy.java index d264e5a6d..9e5e9691e 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/AbstractStorageProxy.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/AbstractStorageProxy.java @@ -139,6 +139,11 @@ public Blob get(BlobId blob) { return delegate.get(blob); } + @Override + public Blob restore(BlobId blob, BlobRestoreOption... options) { + return delegate.restore(blob, options); + } + @Override public Page list(BucketListOption... options) { return delegate.list(options);