Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add soft delete feature #2403

Merged
merged 30 commits into from
Mar 14, 2024
Merged
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
26326a4
feat: add soft delete feature
JesseLovelace Feb 8, 2024
407af36
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Feb 8, 2024
22ae88d
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Feb 8, 2024
bd4bb67
Merge branch 'softdel' of https://github.com/googleapis/java-storage …
gcf-owl-bot[bot] Feb 8, 2024
d77aa58
add softDeleteTime and hardDeleteTime object fields
JesseLovelace Feb 9, 2024
a888e80
Merge branch 'softdel' of github.com:googleapis/java-storage into sof…
JesseLovelace Feb 9, 2024
2a85a65
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Feb 9, 2024
4f1b6de
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Feb 9, 2024
fd22008
add new fields to field tests
JesseLovelace Feb 9, 2024
7142bcb
Merge branch 'softdel' of github.com:googleapis/java-storage into sof…
JesseLovelace Feb 9, 2024
0f837d3
clirr ignore
JesseLovelace Feb 9, 2024
c2884aa
remove debug comments
JesseLovelace Feb 14, 2024
ac6022e
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Feb 14, 2024
1c363ff
Merge branch 'main' into softdel
JesseLovelace Feb 28, 2024
dbe0d92
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Feb 28, 2024
fb88608
add softdeletetime and harddeletetime to grpc codec
JesseLovelace Feb 28, 2024
04975fc
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Feb 28, 2024
1de29b8
fix read mask test
JesseLovelace Feb 29, 2024
c98d8c7
Merge branch 'softdel' of github.com:googleapis/java-storage into sof…
JesseLovelace Feb 29, 2024
3a4a00f
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Feb 29, 2024
b3d70db
fix style issues
JesseLovelace Feb 29, 2024
7986f9c
Merge branch 'softdel' of github.com:googleapis/java-storage into sof…
JesseLovelace Feb 29, 2024
a93e162
Merge branch 'main' into softdel
JesseLovelace Mar 11, 2024
5a744ca
updates to apiary library
JesseLovelace Mar 11, 2024
0b8fe3f
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Mar 11, 2024
be3e975
Merge branch 'main' into softdel
JesseLovelace Mar 14, 2024
5e65c40
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Mar 14, 2024
938e00c
fix read mask test
JesseLovelace Mar 14, 2024
8f38221
Merge branch 'softdel' of github.com:googleapis/java-storage into sof…
JesseLovelace Mar 14, 2024
68ce8c0
fix typo
JesseLovelace Mar 14, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
18 changes: 18 additions & 0 deletions google-cloud-storage/clirr-ignored-differences.xml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,24 @@
<method>com.google.cloud.storage.BlobInfo$Builder setRetention(com.google.cloud.storage.BlobInfo$Retention)</method>
</difference>

<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/storage/BucketInfo$Builder</className>
<method>com.google.cloud.storage.BucketInfo$Builder setSoftDeletePolicy(com.google.cloud.storage.BucketInfo$SoftDeletePolicy)</method>
</difference>

<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/storage/Storage</className>
<method>com.google.cloud.storage.Blob restore(com.google.cloud.storage.BlobId, com.google.cloud.storage.Storage$BlobRestoreOption[])</method>
</difference>

<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/storage/spi/v1/StorageRpc</className>
<method>com.google.api.services.storage.model.StorageObject restore(com.google.api.services.storage.model.StorageObject, java.util.Map)</method>
</difference>

<!-- @BetaApi members -->
<difference>
<differenceType>7009</differenceType>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -526,6 +526,18 @@ Builder setRetentionExpirationTimeOffsetDateTime(OffsetDateTime retentionExpirat
return this;
}

@Override
Builder setSoftDeleteTime(OffsetDateTime softDeleteTime) {
infoBuilder.setSoftDeleteTime(softDeleteTime);
return this;
}

@Override
Builder setHardDeleteTime(OffsetDateTime hardDeleteTime) {
infoBuilder.setHardDeleteTime(hardDeleteTime);
return this;
}

@Override
public Builder setRetention(Retention retention) {
infoBuilder.setRetention(retention);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ public class BlobInfo implements Serializable {
private final Boolean temporaryHold;
private final OffsetDateTime retentionExpirationTime;
private final Retention retention;
private final OffsetDateTime softDeleteTime;
private final OffsetDateTime hardDeleteTime;
private final transient ImmutableSet<NamedField> modifiedFields;

/** This class is meant for internal use only. Users are discouraged from using this class. */
Expand Down Expand Up @@ -525,6 +527,10 @@ Builder setRetentionExpirationTimeOffsetDateTime(OffsetDateTime retentionExpirat
return setRetentionExpirationTime(millisOffsetDateTimeCodec.decode(retentionExpirationTime));
}

abstract Builder setSoftDeleteTime(OffsetDateTime offsetDateTime);

abstract Builder setHardDeleteTime(OffsetDateTime hardDeleteTIme);

public abstract Builder setRetention(Retention retention);

/** Creates a {@code BlobInfo} object. */
Expand Down Expand Up @@ -626,6 +632,8 @@ static final class BuilderImpl extends Builder {
private Boolean temporaryHold;
private OffsetDateTime retentionExpirationTime;
private Retention retention;
private OffsetDateTime softDeleteTime;
private OffsetDateTime hardDeleteTime;
private final ImmutableSet.Builder<NamedField> modifiedFields = ImmutableSet.builder();

BuilderImpl(BlobId blobId) {
Expand Down Expand Up @@ -664,6 +672,8 @@ static final class BuilderImpl extends Builder {
temporaryHold = blobInfo.temporaryHold;
retentionExpirationTime = blobInfo.retentionExpirationTime;
retention = blobInfo.retention;
softDeleteTime = blobInfo.softDeleteTime;
hardDeleteTime = blobInfo.hardDeleteTime;
}

@Override
Expand Down Expand Up @@ -1037,6 +1047,24 @@ Builder setRetentionExpirationTimeOffsetDateTime(OffsetDateTime retentionExpirat
return this;
}

@Override
Builder setSoftDeleteTime(OffsetDateTime softDeleteTime) {
if (!Objects.equals(this.softDeleteTime, softDeleteTime)) {
modifiedFields.add(BlobField.SOFT_DELETE_TIME);
}
this.softDeleteTime = softDeleteTime;
return this;
}

@Override
Builder setHardDeleteTime(OffsetDateTime hardDeleteTime) {
if (!Objects.equals(this.hardDeleteTime, hardDeleteTime)) {
modifiedFields.add(BlobField.HARD_DELETE_TIME);
}
this.hardDeleteTime = hardDeleteTime;
return this;
}

@Override
public Builder setRetention(Retention retention) {
// todo: b/308194853
Expand Down Expand Up @@ -1269,6 +1297,8 @@ Builder clearRetentionExpirationTime() {
temporaryHold = builder.temporaryHold;
retentionExpirationTime = builder.retentionExpirationTime;
retention = builder.retention;
softDeleteTime = builder.softDeleteTime;
hardDeleteTime = builder.hardDeleteTime;
modifiedFields = builder.modifiedFields.build();
}

Expand Down Expand Up @@ -1662,6 +1692,18 @@ public OffsetDateTime getRetentionExpirationTimeOffsetDateTime() {
return retentionExpirationTime;
}

/** If this object has been soft-deleted, returns the time it was soft-deleted. */
public OffsetDateTime getSoftDeleteTime() {
return softDeleteTime;
}

/**
* If this object has been soft-deleted, returns the time at which it will be permanently deleted.
*/
public OffsetDateTime getHardDeleteTime() {
return hardDeleteTime;
}

/** Returns the object's Retention policy. */
public Retention getRetention() {
return retention;
Expand Down Expand Up @@ -1717,7 +1759,9 @@ public int hashCode() {
eventBasedHold,
temporaryHold,
retention,
retentionExpirationTime);
retentionExpirationTime,
softDeleteTime,
hardDeleteTime);
}

@Override
Expand Down Expand Up @@ -1759,7 +1803,9 @@ public boolean equals(Object o) {
&& Objects.equals(eventBasedHold, blobInfo.eventBasedHold)
&& Objects.equals(temporaryHold, blobInfo.temporaryHold)
&& Objects.equals(retentionExpirationTime, blobInfo.retentionExpirationTime)
&& Objects.equals(retention, blobInfo.retention);
&& Objects.equals(retention, blobInfo.retention)
&& Objects.equals(softDeleteTime, blobInfo.softDeleteTime)
&& Objects.equals(hardDeleteTime, blobInfo.hardDeleteTime);
}

ImmutableSet<NamedField> getModifiedFields() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,12 @@ Builder setObjectRetention(ObjectRetention objectRetention) {
return this;
}

@Override
public Builder setSoftDeletePolicy(SoftDeletePolicy softDeletePolicy) {
infoBuilder.setSoftDeletePolicy(softDeletePolicy);
return this;
}

@Override
public Bucket build() {
return new Bucket(storage, infoBuilder);
Expand Down Expand Up @@ -1083,6 +1089,28 @@ public Blob get(String blob, BlobGetOption... options) {
return storage.get(BlobId.of(getName(), blob), options);
}

/**
* Returns the requested blob in this bucket of a specific generation or {@code null} if not
* found.
*
* <p>Example of getting a blob of a specific in the bucket.
*
* <pre>{@code
* String blobName = "my_blob_name";
* long generation = 42;
* Blob blob = bucket.get(blobName, generation);
* }</pre>
*
* @param blob name of the requested blob
* @param generation the generation to get
* @param options blob search options
* @throws StorageException upon failure
*/
@TransportCompatibility({Transport.HTTP, Transport.GRPC})
public Blob get(String blob, Long generation, BlobGetOption... options) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IIUC this is needed because you can't specify generation on Bucket.get() that exists today; Would recommend using BlobId instead of adding Long generation overload.

Do you think this is hard requirement to support this feature since there's storage.get(BlobId..)?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it would feel pretty weird that soft deleted would become the only state an object can be in where you can get it from Storage but can't get it from the Bucket that the object lives in.

I also think using BlobId here would feel weird, because you're already calling from a Bucket, so it's weird to have to supply the same Bucket name into the BlobId. There are no other methods in Bucket.java that have a BlobId in the signature

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bleh, missed that bucket name is also part of BlobId. thanks.
We haven't had generation in Bucket.get() at all which is required for version enabled buckets, that's why i asked this question as well.

For the sake of discussion, WDYT about having generation under BlobGetOption's? It's a query paramter just like IfGenerationMatch etc.

return storage.get(BlobId.of(getName(), blob, generation), options);
}

/**
* Returns a list of requested blobs in this bucket. Blobs that do not exist are null.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ public class BucketInfo implements Serializable {
private final CustomPlacementConfig customPlacementConfig;
private final ObjectRetention objectRetention;

private final SoftDeletePolicy softDeletePolicy;

private final transient ImmutableSet<NamedField> modifiedFields;

/**
Expand Down Expand Up @@ -349,6 +351,90 @@ public String toString() {
}
}

/**
* The bucket's soft delete policy. If this policy is set, any deleted objects will be
* soft-deleted according to the time specified in the policy
*/
public static class SoftDeletePolicy implements Serializable {

private static final long serialVersionUID = -8100190443052242908L;
private Duration retentionDuration;
private OffsetDateTime effectiveTime;

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof SoftDeletePolicy)) {
return false;
}
SoftDeletePolicy that = (SoftDeletePolicy) o;
return Objects.equals(retentionDuration, that.retentionDuration)
&& Objects.equals(effectiveTime, that.effectiveTime);
}

@Override
public int hashCode() {
return Objects.hash(retentionDuration, effectiveTime);
}

@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("retentionDuration", retentionDuration)
.add("effectiveTime", effectiveTime)
.toString();
}

public static Builder newBuilder() {
return new Builder();
}

public Builder toBuilder() {
return new Builder().setRetentionDuration(retentionDuration).setEffectiveTime(effectiveTime);
}

private SoftDeletePolicy() {}

public SoftDeletePolicy(Builder builder) {
this.retentionDuration = builder.retentionDuration;
this.effectiveTime = builder.effectiveTime;
}

public Duration getRetentionDuration() {
return retentionDuration;
}

public OffsetDateTime getEffectiveTime() {
return effectiveTime;
}

public static final class Builder {
private Duration retentionDuration;
private OffsetDateTime effectiveTime;

/** Sets the length of time to retain soft-deleted objects for, expressed as a Duration */
public Builder setRetentionDuration(Duration retentionDuration) {
this.retentionDuration = retentionDuration;
return this;
}

/**
* Sets the time from which this soft-delete policy is effective. This is package-private
* because it can only be set by the backend.
*/
Builder setEffectiveTime(OffsetDateTime effectiveTime) {
this.effectiveTime = effectiveTime;
return this;
}

public SoftDeletePolicy build() {
return new SoftDeletePolicy(this);
}
}
}

/**
* Configuration for the Autoclass settings of a bucket.
*
Expand Down Expand Up @@ -1685,6 +1771,8 @@ public Builder setRetentionPeriodDuration(Duration retentionPeriod) {

abstract Builder setObjectRetention(ObjectRetention objectRetention);

public abstract Builder setSoftDeletePolicy(SoftDeletePolicy softDeletePolicy);

/** Creates a {@code BucketInfo} object. */
public abstract BucketInfo build();

Expand Down Expand Up @@ -1783,6 +1871,8 @@ static final class BuilderImpl extends Builder {
private Logging logging;
private CustomPlacementConfig customPlacementConfig;
private ObjectRetention objectRetention;

private SoftDeletePolicy softDeletePolicy;
private final ImmutableSet.Builder<NamedField> modifiedFields = ImmutableSet.builder();

BuilderImpl(String name) {
Expand Down Expand Up @@ -1822,6 +1912,7 @@ static final class BuilderImpl extends Builder {
logging = bucketInfo.logging;
customPlacementConfig = bucketInfo.customPlacementConfig;
objectRetention = bucketInfo.objectRetention;
softDeletePolicy = bucketInfo.softDeletePolicy;
}

@Override
Expand Down Expand Up @@ -2187,6 +2278,15 @@ Builder setObjectRetention(ObjectRetention objectRetention) {
return this;
}

@Override
public Builder setSoftDeletePolicy(SoftDeletePolicy softDeletePolicy) {
if (!Objects.equals(this.softDeletePolicy, softDeletePolicy)) {
modifiedFields.add(BucketField.SOFT_DELETE_POLICY);
}
this.softDeletePolicy = softDeletePolicy;
return this;
}

@Override
Builder setLocationType(String locationType) {
if (!Objects.equals(this.locationType, locationType)) {
Expand Down Expand Up @@ -2428,6 +2528,7 @@ private Builder clearDeleteLifecycleRules() {
logging = builder.logging;
customPlacementConfig = builder.customPlacementConfig;
objectRetention = builder.objectRetention;
softDeletePolicy = builder.softDeletePolicy;
modifiedFields = builder.modifiedFields.build();
}

Expand Down Expand Up @@ -2768,6 +2869,11 @@ public ObjectRetention getObjectRetention() {
return objectRetention;
}

/** returns the Soft Delete policy */
public SoftDeletePolicy getSoftDeletePolicy() {
return softDeletePolicy;
}

/** Returns a builder for the current bucket. */
public Builder toBuilder() {
return new BuilderImpl(this);
Expand Down Expand Up @@ -2805,6 +2911,7 @@ public int hashCode() {
autoclass,
locationType,
objectRetention,
softDeletePolicy,
logging);
}

Expand Down Expand Up @@ -2846,6 +2953,7 @@ public boolean equals(Object o) {
&& Objects.equals(autoclass, that.autoclass)
&& Objects.equals(locationType, that.locationType)
&& Objects.equals(objectRetention, that.objectRetention)
&& Objects.equals(softDeletePolicy, that.softDeletePolicy)
&& Objects.equals(logging, that.logging);
}

Expand Down