Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add hierarchical namespace and folders features #2445

Merged
merged 11 commits into from
Mar 14, 2024
6 changes: 6 additions & 0 deletions google-cloud-storage/clirr-ignored-differences.xml
Expand Up @@ -15,6 +15,12 @@
<method>* writeAndClose(*)</method>
</difference>

<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/storage/BucketInfo$Builder</className>
<method>com.google.cloud.storage.BucketInfo$Builder setHierarchicalNamespace(com.google.cloud.storage.BucketInfo$HierarchicalNamespace)</method>
</difference>

<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/storage/BlobInfo$Builder</className>
Expand Down
Expand Up @@ -748,6 +748,12 @@ Builder setObjectRetention(ObjectRetention objectRetention) {
return this;
}

@Override
public Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace) {
infoBuilder.setHierarchicalNamespace(hierarchicalNamespace);
return this;
}

@Override
public Bucket build() {
return new Bucket(storage, infoBuilder);
Expand Down
Expand Up @@ -118,6 +118,7 @@ public class BucketInfo implements Serializable {
private final Logging logging;
private final CustomPlacementConfig customPlacementConfig;
private final ObjectRetention objectRetention;
private final HierarchicalNamespace hierarchicalNamespace;

private final transient ImmutableSet<NamedField> modifiedFields;

Expand Down Expand Up @@ -713,6 +714,71 @@ public Logging build() {
}
}

/** The bucket's hierarchical namespace (Folders) configuration. Enable this to use HNS. */
public static final class HierarchicalNamespace implements Serializable {

private static final long serialVersionUID = 5932926691444613101L;
private Boolean enabled;

public Boolean getEnabled() {
return enabled;
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof HierarchicalNamespace)) {
return false;
}
HierarchicalNamespace that = (HierarchicalNamespace) o;
return Objects.equals(enabled, that.enabled);
}

@Override
public int hashCode() {
return Objects.hash(enabled);
}

@Override
public String toString() {
return MoreObjects.toStringHelper(this).add("enabled", enabled).toString();
}

private HierarchicalNamespace() {}

private HierarchicalNamespace(Builder builder) {
this.enabled = builder.enabled;
}

public static Builder newBuilder() {
return new Builder();
}

public Builder toBuilder() {
return newBuilder().setEnabled(enabled);
}

public static final class Builder {
private Boolean enabled;

/**
* Sets whether Hierarchical Namespace (Folders) is enabled for this bucket. This can only be
* enabled at bucket create time. If this is enabled, Uniform Bucket-Level Access must also be
* enabled.
*/
public Builder setEnabled(Boolean enabled) {
this.enabled = enabled;
return this;
}

public HierarchicalNamespace build() {
return new HierarchicalNamespace(this);
}
}
}

/**
* Lifecycle rule for a bucket. Allows supported Actions, such as deleting and changing storage
* class, to be executed when certain Conditions are met.
Expand Down Expand Up @@ -1683,6 +1749,8 @@ public Builder setRetentionPeriodDuration(Duration retentionPeriod) {

public abstract Builder setCustomPlacementConfig(CustomPlacementConfig customPlacementConfig);

public abstract Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace);

abstract Builder setObjectRetention(ObjectRetention objectRetention);

/** Creates a {@code BucketInfo} object. */
Expand Down Expand Up @@ -1783,6 +1851,7 @@ static final class BuilderImpl extends Builder {
private Logging logging;
private CustomPlacementConfig customPlacementConfig;
private ObjectRetention objectRetention;
private HierarchicalNamespace hierarchicalNamespace;
private final ImmutableSet.Builder<NamedField> modifiedFields = ImmutableSet.builder();

BuilderImpl(String name) {
Expand Down Expand Up @@ -1822,6 +1891,7 @@ static final class BuilderImpl extends Builder {
logging = bucketInfo.logging;
customPlacementConfig = bucketInfo.customPlacementConfig;
objectRetention = bucketInfo.objectRetention;
hierarchicalNamespace = bucketInfo.hierarchicalNamespace;
}

@Override
Expand Down Expand Up @@ -2187,6 +2257,15 @@ Builder setObjectRetention(ObjectRetention objectRetention) {
return this;
}

@Override
public Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace) {
if (!Objects.equals(this.hierarchicalNamespace, hierarchicalNamespace)) {
modifiedFields.add(BucketField.HIERARCHICAL_NAMESPACE);
}
this.hierarchicalNamespace = hierarchicalNamespace;
return this;
}

@Override
Builder setLocationType(String locationType) {
if (!Objects.equals(this.locationType, locationType)) {
Expand Down Expand Up @@ -2428,6 +2507,7 @@ private Builder clearDeleteLifecycleRules() {
logging = builder.logging;
customPlacementConfig = builder.customPlacementConfig;
objectRetention = builder.objectRetention;
hierarchicalNamespace = builder.hierarchicalNamespace;
modifiedFields = builder.modifiedFields.build();
}

Expand Down Expand Up @@ -2768,6 +2848,11 @@ public ObjectRetention getObjectRetention() {
return objectRetention;
}

/** Returns the Hierarchical Namespace (Folders) Configuration */
public HierarchicalNamespace getHierarchicalNamespace() {
return hierarchicalNamespace;
}

/** Returns a builder for the current bucket. */
public Builder toBuilder() {
return new BuilderImpl(this);
Expand Down Expand Up @@ -2805,6 +2890,7 @@ public int hashCode() {
autoclass,
locationType,
objectRetention,
hierarchicalNamespace,
logging);
}

Expand Down Expand Up @@ -2846,6 +2932,7 @@ public boolean equals(Object o) {
&& Objects.equals(autoclass, that.autoclass)
&& Objects.equals(locationType, that.locationType)
&& Objects.equals(objectRetention, that.objectRetention)
&& Objects.equals(hierarchicalNamespace, that.hierarchicalNamespace)
&& Objects.equals(logging, that.logging);
}

Expand Down
Expand Up @@ -110,6 +110,10 @@ final class GrpcConversions {
private final Codec<Condition, Expr> iamConditionCodec =
Codec.of(this::conditionEncode, this::conditionDecode);

private final Codec<BucketInfo.HierarchicalNamespace, Bucket.HierarchicalNamespace>
hierarchicalNamespaceCodec =
Codec.of(this::hierarchicalNamespaceEncode, this::hierarchicalNamespaceDecode);

@VisibleForTesting
final Codec<OffsetDateTime, Timestamp> timestampCodec =
Codec.of(
Expand Down Expand Up @@ -297,6 +301,10 @@ private BucketInfo bucketInfoDecode(Bucket from) {
.setDataLocations(customPlacementConfig.getDataLocationsList())
.build());
}
if (from.hasHierarchicalNamespace()) {
to.setHierarchicalNamespace(
hierarchicalNamespaceCodec.decode(from.getHierarchicalNamespace()));
}
// TODO(frankyn): Add SelfLink when the field is available
if (!from.getEtag().isEmpty()) {
to.setEtag(from.getEtag());
Expand Down Expand Up @@ -382,6 +390,10 @@ private Bucket bucketInfoEncode(BucketInfo from) {
.addAllDataLocations(customPlacementConfig.getDataLocations())
.build());
}
ifNonNull(
from.getHierarchicalNamespace(),
hierarchicalNamespaceCodec::encode,
to::setHierarchicalNamespace);
// TODO(frankyn): Add SelfLink when the field is available
ifNonNull(from.getEtag(), to::setEtag);
return to.build();
Expand Down Expand Up @@ -589,6 +601,20 @@ private Bucket.Autoclass autoclassEncode(BucketInfo.Autoclass from) {
return to.build();
}

private Bucket.HierarchicalNamespace hierarchicalNamespaceEncode(
BucketInfo.HierarchicalNamespace from) {
Bucket.HierarchicalNamespace.Builder to = Bucket.HierarchicalNamespace.newBuilder();
ifNonNull(from.getEnabled(), to::setEnabled);
return to.build();
}

private BucketInfo.HierarchicalNamespace hierarchicalNamespaceDecode(
Bucket.HierarchicalNamespace from) {
BucketInfo.HierarchicalNamespace.Builder to = BucketInfo.HierarchicalNamespace.newBuilder();
to.setEnabled(from.getEnabled());
return to.build();
}

private Bucket.IamConfig iamConfigEncode(BucketInfo.IamConfiguration from) {
Bucket.IamConfig.Builder to = Bucket.IamConfig.newBuilder();
to.setUniformBucketLevelAccess(ublaEncode(from));
Expand Down
Expand Up @@ -137,6 +137,10 @@ final class JsonConversions {
private final Codec<BlobInfo, StorageObject> blobInfoCodec =
Codec.of(this::blobInfoEncode, this::blobInfoDecode);

private final Codec<BucketInfo.HierarchicalNamespace, Bucket.HierarchicalNamespace>
hierarchicalNamespaceCodec =
Codec.of(this::hierarchicalNamespaceEncode, this::hierarchicalNamespaceDecode);

private final Codec<NotificationInfo, com.google.api.services.storage.model.Notification>
notificationInfoCodec = Codec.of(this::notificationEncode, this::notificationDecode);
private final Codec<CustomPlacementConfig, Bucket.CustomPlacementConfig>
Expand Down Expand Up @@ -437,6 +441,10 @@ private Bucket bucketInfoEncode(BucketInfo from) {
this::customPlacementConfigEncode,
to::setCustomPlacementConfig);
ifNonNull(from.getObjectRetention(), this::objectRetentionEncode, to::setObjectRetention);
ifNonNull(
from.getHierarchicalNamespace(),
this::hierarchicalNamespaceEncode,
to::setHierarchicalNamespace);
return to;
}

Expand Down Expand Up @@ -487,6 +495,10 @@ private BucketInfo bucketInfoDecode(com.google.api.services.storage.model.Bucket
from.getCustomPlacementConfig(),
this::customPlacementConfigDecode,
to::setCustomPlacementConfig);
ifNonNull(
from.getHierarchicalNamespace(),
this::hierarchicalNamespaceDecode,
to::setHierarchicalNamespace);
ifNonNull(from.getObjectRetention(), this::objectRetentionDecode, to::setObjectRetention);
return to.build();
}
Expand Down Expand Up @@ -861,6 +873,20 @@ private com.google.api.services.storage.model.Notification notificationEncode(
return to;
}

private Bucket.HierarchicalNamespace hierarchicalNamespaceEncode(
BucketInfo.HierarchicalNamespace from) {
Bucket.HierarchicalNamespace to = new Bucket.HierarchicalNamespace();
ifNonNull(from.getEnabled(), to::setEnabled);
return to;
}

private BucketInfo.HierarchicalNamespace hierarchicalNamespaceDecode(
Bucket.HierarchicalNamespace from) {
BucketInfo.HierarchicalNamespace.Builder to = BucketInfo.HierarchicalNamespace.newBuilder();
to.setEnabled(from.getEnabled());
return to.build();
}

private NotificationInfo notificationDecode(
com.google.api.services.storage.model.Notification from) {
NotificationInfo.Builder builder = new NotificationInfo.BuilderImpl(from.getTopic());
Expand Down
Expand Up @@ -159,6 +159,9 @@ enum BucketField implements FieldSelector, NamedField {
CUSTOM_PLACEMENT_CONFIG("customPlacementConfig", "custom_placement_config"),
@TransportCompatibility({Transport.HTTP, Transport.GRPC})
AUTOCLASS("autoclass"),

@TransportCompatibility({Transport.HTTP, Transport.GRPC})
HIERARCHICAL_NAMESPACE("hierarchicalNamespace", "hierarchical_namespace"),
@TransportCompatibility({Transport.HTTP})
OBJECT_RETENTION("objectRetention");

Expand Down Expand Up @@ -1788,6 +1791,14 @@ public static BlobListOption matchGlob(@NonNull String glob) {
return new BlobListOption(UnifiedOpts.matchGlob(glob));
}

/**
* Returns an option for whether to include all Folders (including empty Folders) in response.
*/
@TransportCompatibility({Transport.HTTP, Transport.GRPC})
public static BlobListOption includeFolders(boolean includeFolders) {
return new BlobListOption(UnifiedOpts.includeFoldersAsPrefixes(includeFolders));
}

/**
* Returns an option to define the billing user project. This option is required by buckets with
* `requester_pays` flag enabled to assign operation costs.
Expand Down
Expand Up @@ -370,6 +370,10 @@ static Delimiter delimiter(@NonNull String delimiter) {
return new Delimiter(delimiter);
}

static IncludeFoldersAsPrefixes includeFoldersAsPrefixes(boolean includeFoldersAsPrefixes) {
return new IncludeFoldersAsPrefixes(includeFoldersAsPrefixes);
}

@Deprecated
static DetectContentType detectContentType() {
return DetectContentType.INSTANCE;
Expand Down Expand Up @@ -636,6 +640,20 @@ public Mapper<RewriteObjectRequest.Builder> rewriteObject() {
}
}

static final class IncludeFoldersAsPrefixes extends RpcOptVal<Boolean> implements ObjectListOpt {

private static final long serialVersionUID = 321916692864878282L;

private IncludeFoldersAsPrefixes(boolean val) {
super(StorageRpc.Option.INCLUDE_FOLDERS_AS_PREFIXES, val);
}

@Override
public Mapper<ListObjectsRequest.Builder> listObjects() {
return b -> b.setIncludeFoldersAsPrefixes(val);
}
}

static final class Delimiter extends RpcOptVal<String> implements ObjectListOpt {
private static final long serialVersionUID = -3789556789947615714L;

Expand Down
Expand Up @@ -459,6 +459,7 @@ public Tuple<String, Iterable<StorageObject>> list(final String bucket, Map<Opti
.setPageToken(Option.PAGE_TOKEN.getString(options))
.setFields(Option.FIELDS.getString(options))
.setUserProject(Option.USER_PROJECT.getString(options))
.setIncludeFoldersAsPrefixes(Option.INCLUDE_FOLDERS_AS_PREFIXES.getBoolean(options))
.execute();
Iterable<StorageObject> storageObjects =
Iterables.concat(
Expand Down
Expand Up @@ -73,7 +73,9 @@ enum Option {
DETECT_CONTENT_TYPE("detectContentType"),
ENABLE_OBJECT_RETENTION("enableObjectRetention"),
RETURN_RAW_INPUT_STREAM("returnRawInputStream"),
OVERRIDE_UNLOCKED_RETENTION("overrideUnlockedRetention");
OVERRIDE_UNLOCKED_RETENTION("overrideUnlockedRetention"),
INCLUDE_FOLDERS_AS_PREFIXES("includeFoldersAsPrefixes");
;

private final String value;

Expand Down
Expand Up @@ -133,6 +133,7 @@ public ImmutableList<?> parameters() {
new Args<>(BucketField.TIME_CREATED, LazyAssertion.equal()),
new Args<>(BucketField.UPDATED, LazyAssertion.equal()),
new Args<>(BucketField.VERSIONING, LazyAssertion.equal()),
new Args<>(BucketField.HIERARCHICAL_NAMESPACE, LazyAssertion.equal()),
new Args<>(BucketField.WEBSITE, LazyAssertion.equal()));

List<String> argsDefined =
Expand Down