Add grpc and HNS

googleapis · JesseLovelace · Mar 14, 2024 · Feb 14, 2024 · Mar 12, 2024 · Mar 12, 2024
commit bd65ab03212f7bc98ed71fb2756dd2798105dd6d
@@ -748,6 +748,12 @@ Builder setObjectRetention(ObjectRetention objectRetention) {
  return this;
  }
 
+ @Override
+ public Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace) {
+ infoBuilder.setHierarchicalNamespace(hierarchicalNamespace);
+ return this;
+ }
+
  @Override
  public Bucket build() {
  return new Bucket(storage, infoBuilder);

@@ -118,6 +118,7 @@ public class BucketInfo implements Serializable {
  private final Logging logging;
  private final CustomPlacementConfig customPlacementConfig;
  private final ObjectRetention objectRetention;
+ private final HierarchicalNamespace hierarchicalNamespace;
 
  private final transient ImmutableSet<NamedField> modifiedFields;
 
@@ -713,6 +714,70 @@ public Logging build() {
  }
  }
 
+ public static final class HierarchicalNamespace implements Serializable {
+
+ private static final long serialVersionUID = 5932926691444613101L;
+ private Boolean enabled;
+
+ public Boolean getEnabled() {
+ return enabled;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof HierarchicalNamespace)) {
+ return false;
+ }
+ HierarchicalNamespace that = (HierarchicalNamespace) o;
+ return Objects.equals(enabled, that.enabled);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(enabled);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this).add("enabled", enabled).toString();
+ }
+
+ private HierarchicalNamespace() {}
+
+ private HierarchicalNamespace(Builder builder) {
+ this.enabled = builder.enabled;
+ }
+
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ public Builder toBuilder() {
+ return newBuilder().setEnabled(enabled);
+ }
+
+ public static final class Builder {
+ private Boolean enabled;
+
+ /**
+ * Sets whether Hierarchical Namespace (Folders) is enabled for this bucket. This can only be
+ * enabled at bucket create time. If this is enabled, Uniform Bucket-Level Access must also be
+ * enabled.
+ */
+ public Builder setEnabled(Boolean enabled) {
+ this.enabled = enabled;
+ return this;
+ }
+
+ public HierarchicalNamespace build() {
+ return new HierarchicalNamespace(this);
+ }
+ }
+ }
+
  /**
  * Lifecycle rule for a bucket. Allows supported Actions, such as deleting and changing storage
  * class, to be executed when certain Conditions are met.
@@ -1683,6 +1748,8 @@ public Builder setRetentionPeriodDuration(Duration retentionPeriod) {
 
  public abstract Builder setCustomPlacementConfig(CustomPlacementConfig customPlacementConfig);
 
+ public abstract Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace);
+
  abstract Builder setObjectRetention(ObjectRetention objectRetention);
 
  /** Creates a {@code BucketInfo} object. */
@@ -1783,6 +1850,7 @@ static final class BuilderImpl extends Builder {
  private Logging logging;
  private CustomPlacementConfig customPlacementConfig;
  private ObjectRetention objectRetention;
+ private HierarchicalNamespace hierarchicalNamespace;
  private final ImmutableSet.Builder<NamedField> modifiedFields = ImmutableSet.builder();
 
  BuilderImpl(String name) {
@@ -1822,6 +1890,7 @@ static final class BuilderImpl extends Builder {
  logging = bucketInfo.logging;
  customPlacementConfig = bucketInfo.customPlacementConfig;
  objectRetention = bucketInfo.objectRetention;
+ hierarchicalNamespace = bucketInfo.hierarchicalNamespace;
  }
 
  @Override
@@ -2187,6 +2256,15 @@ Builder setObjectRetention(ObjectRetention objectRetention) {
  return this;
  }
 
+ @Override
+ public Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace) {
+ if (!Objects.equals(this.hierarchicalNamespace, hierarchicalNamespace)) {
+ modifiedFields.add(BucketField.HIERARCHICAL_NAMESPACE);
+ }
+ this.hierarchicalNamespace = hierarchicalNamespace;
+ return this;
+ }
+
  @Override
  Builder setLocationType(String locationType) {
  if (!Objects.equals(this.locationType, locationType)) {
@@ -2428,6 +2506,7 @@ private Builder clearDeleteLifecycleRules() {
  logging = builder.logging;
  customPlacementConfig = builder.customPlacementConfig;
  objectRetention = builder.objectRetention;
+ hierarchicalNamespace = builder.hierarchicalNamespace;
  modifiedFields = builder.modifiedFields.build();
  }
 
@@ -2768,6 +2847,11 @@ public ObjectRetention getObjectRetention() {
  return objectRetention;
  }
 
+ /** Returns the Hierarchical Namespace (Folders) Configuration */
+ public HierarchicalNamespace getHierarchicalNamespace() {
+ return hierarchicalNamespace;
+ }
+
  /** Returns a builder for the current bucket. */
  public Builder toBuilder() {
  return new BuilderImpl(this);
@@ -2805,6 +2889,7 @@ public int hashCode() {
  autoclass,
  locationType,
  objectRetention,
+ hierarchicalNamespace,
  logging);
  }
 
@@ -2846,6 +2931,7 @@ public boolean equals(Object o) {
  && Objects.equals(autoclass, that.autoclass)
  && Objects.equals(locationType, that.locationType)
  && Objects.equals(objectRetention, that.objectRetention)
+ && Objects.equals(hierarchicalNamespace, that.hierarchicalNamespace)
  && Objects.equals(logging, that.logging);
  }
 

@@ -110,6 +110,10 @@ final class GrpcConversions {
  private final Codec<Condition, Expr> iamConditionCodec =
  Codec.of(this::conditionEncode, this::conditionDecode);
 
+ private final Codec<BucketInfo.HierarchicalNamespace, Bucket.HierarchicalNamespace>
+ hierarchicalNamespaceCodec =
+ Codec.of(this::hierarchicalNamespaceEncode, this::hierarchicalNamespaceDecode);
+
  @VisibleForTesting
  final Codec<OffsetDateTime, Timestamp> timestampCodec =
  Codec.of(
@@ -297,6 +301,10 @@ private BucketInfo bucketInfoDecode(Bucket from) {
  .setDataLocations(customPlacementConfig.getDataLocationsList())
  .build());
  }
+ if (from.hasHierarchicalNamespace()) {
+ to.setHierarchicalNamespace(
+ hierarchicalNamespaceCodec.decode(from.getHierarchicalNamespace()));
+ }
  // TODO(frankyn): Add SelfLink when the field is available
  if (!from.getEtag().isEmpty()) {
  to.setEtag(from.getEtag());
@@ -382,6 +390,10 @@ private Bucket bucketInfoEncode(BucketInfo from) {
  .addAllDataLocations(customPlacementConfig.getDataLocations())
  .build());
  }
+ ifNonNull(
+ from.getHierarchicalNamespace(),
+ hierarchicalNamespaceCodec::encode,
+ to::setHierarchicalNamespace);
  // TODO(frankyn): Add SelfLink when the field is available
  ifNonNull(from.getEtag(), to::setEtag);
  return to.build();
@@ -589,6 +601,20 @@ private Bucket.Autoclass autoclassEncode(BucketInfo.Autoclass from) {
  return to.build();
  }
 
+ private Bucket.HierarchicalNamespace hierarchicalNamespaceEncode(
+ BucketInfo.HierarchicalNamespace from) {
+ Bucket.HierarchicalNamespace.Builder to = Bucket.HierarchicalNamespace.newBuilder();
+ ifNonNull(from.getEnabled(), to::setEnabled);
+ return to.build();
+ }
+
+ private BucketInfo.HierarchicalNamespace hierarchicalNamespaceDecode(
+ Bucket.HierarchicalNamespace from) {
+ BucketInfo.HierarchicalNamespace.Builder to = BucketInfo.HierarchicalNamespace.newBuilder();
+ to.setEnabled(from.getEnabled());
+ return to.build();
+ }
+
  private Bucket.IamConfig iamConfigEncode(BucketInfo.IamConfiguration from) {
  Bucket.IamConfig.Builder to = Bucket.IamConfig.newBuilder();
  to.setUniformBucketLevelAccess(ublaEncode(from));

@@ -137,6 +137,10 @@ final class JsonConversions {
  private final Codec<BlobInfo, StorageObject> blobInfoCodec =
  Codec.of(this::blobInfoEncode, this::blobInfoDecode);
 
+ private final Codec<BucketInfo.HierarchicalNamespace, Bucket.HierarchicalNamespace>
+ hierarchicalNamespaceCodec =
+ Codec.of(this::hierarchicalNamespaceEncode, this::hierarchicalNamespaceDecode);
+
  private final Codec<NotificationInfo, com.google.api.services.storage.model.Notification>
  notificationInfoCodec = Codec.of(this::notificationEncode, this::notificationDecode);
  private final Codec<CustomPlacementConfig, Bucket.CustomPlacementConfig>
@@ -437,6 +441,10 @@ private Bucket bucketInfoEncode(BucketInfo from) {
  this::customPlacementConfigEncode,
  to::setCustomPlacementConfig);
  ifNonNull(from.getObjectRetention(), this::objectRetentionEncode, to::setObjectRetention);
+ ifNonNull(
+ from.getHierarchicalNamespace(),
+ this::hierarchicalNamespaceEncode,
+ to::setHierarchicalNamespace);
  return to;
  }
 
@@ -487,6 +495,10 @@ private BucketInfo bucketInfoDecode(com.google.api.services.storage.model.Bucket
  from.getCustomPlacementConfig(),
  this::customPlacementConfigDecode,
  to::setCustomPlacementConfig);
+ ifNonNull(
+ from.getHierarchicalNamespace(),
+ this::hierarchicalNamespaceDecode,
+ to::setHierarchicalNamespace);
  ifNonNull(from.getObjectRetention(), this::objectRetentionDecode, to::setObjectRetention);
  return to.build();
  }
@@ -861,6 +873,19 @@ private com.google.api.services.storage.model.Notification notificationEncode(
  return to;
  }
 
+ private Bucket.HierarchicalNamespace hierarchicalNamespaceEncode(BucketInfo.HierarchicalNamespace from) {
+ Bucket.HierarchicalNamespace to = new Bucket.HierarchicalNamespace();
+ ifNonNull(from.getEnabled(), to::setEnabled);
+ return to;
+ }
+
+ private BucketInfo.HierarchicalNamespace hierarchicalNamespaceDecode(Bucket.HierarchicalNamespace from) {
+ BucketInfo.HierarchicalNamespace.Builder to = BucketInfo.HierarchicalNamespace.newBuilder();
+ to.setEnabled(from.getEnabled());
+ return to.build();
+ }
+
+
  private NotificationInfo notificationDecode(
  com.google.api.services.storage.model.Notification from) {
  NotificationInfo.Builder builder = new NotificationInfo.BuilderImpl(from.getTopic());

@@ -159,6 +159,9 @@ enum BucketField implements FieldSelector, NamedField {
  CUSTOM_PLACEMENT_CONFIG("customPlacementConfig", "custom_placement_config"),
  @TransportCompatibility({Transport.HTTP, Transport.GRPC})
  AUTOCLASS("autoclass"),
+
+ @TransportCompatibility({Transport.HTTP, Transport.GRPC})
+ HIERARCHICAL_NAMESPACE("hierarchicalNamespace", "hierarchical_namespace"),
  @TransportCompatibility({Transport.HTTP})
  OBJECT_RETENTION("objectRetention");
 

@@ -651,7 +651,7 @@ private IncludeFoldersAsPrefixes(boolean val) {
 
  @Override
  public Mapper<ListObjectsRequest.Builder> listObjects() {
- return null;// return b -> b.setIncludeFoldersAsPrefixes(val);
+ return b -> b.setIncludeFoldersAsPrefixes(val);
  }
  }
 

@@ -26,6 +26,7 @@
 import static org.junit.Assert.assertTrue;
 
 import com.google.api.gax.paging.Page;
+import com.google.api.services.storage.model.Folder;
 import com.google.cloud.Policy;
 import com.google.cloud.storage.Blob;
 import com.google.cloud.storage.BlobInfo;
@@ -44,6 +45,7 @@
 import com.google.cloud.storage.Storage.BucketListOption;
 import com.google.cloud.storage.Storage.BucketTargetOption;
 import com.google.cloud.storage.StorageClass;
+import com.google.cloud.storage.StorageOptions;
 import com.google.cloud.storage.TestUtils;
 import com.google.cloud.storage.TransportCompatibility.Transport;
 import com.google.cloud.storage.it.runner.StorageITRunner;
@@ -53,6 +55,7 @@
 import com.google.cloud.storage.it.runner.annotations.CrossRun;
 import com.google.cloud.storage.it.runner.annotations.Inject;
 import com.google.cloud.storage.it.runner.registry.Generator;
+import com.google.cloud.storage.spi.v1.HttpStorageRpc;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import java.time.Duration;
@@ -550,6 +553,75 @@ public void testUpdateBucket_noModification() throws Exception {
  assertThat(gen2).isEqualTo(gen1);
  }
  }
+ @Test
+ public void createBucketWithHierarchicalNamespace() {
+ String bucketName = generator.randomBucketName();
+ storage.create(
+ BucketInfo.newBuilder(bucketName)
+ .setHierarchicalNamespace(BucketInfo.HierarchicalNamespace.newBuilder().setEnabled(true).build())
+ .setIamConfiguration(
+ BucketInfo.IamConfiguration.newBuilder()
+ .setIsUniformBucketLevelAccessEnabled(true)
+ .build())
+ .build());
+ try {
+ Bucket remoteBucket = storage.get(bucketName);
+ assertNotNull(remoteBucket.getHierarchicalNamespace());
+ assertTrue(remoteBucket.getHierarchicalNamespace().getEnabled());
+ } finally {
+ BucketCleaner.doCleanup(bucketName, storage);
+ }
+ }
+ @Test
+ public void testListObjectsWithFolders() throws Exception {
+ String bucketName = generator.randomBucketName();
+ storage.create(
+ BucketInfo.newBuilder(bucketName)
+ .setHierarchicalNamespace(BucketInfo.HierarchicalNamespace.newBuilder().setEnabled(true).build())
+ .setIamConfiguration(
+ BucketInfo.IamConfiguration.newBuilder()
+ .setIsUniformBucketLevelAccessEnabled(true)
+ .build())
+ .build());
+ try {
+ com.google.api.services.storage.Storage apiaryStorage =
+ new HttpStorageRpc(StorageOptions.getDefaultInstance()).getStorage();
+ apiaryStorage
+ .folders()
+ .insert(bucketName, new Folder().setName("F").setBucket(bucketName))
+ .execute();
+
+ Page<Blob> blobs =
+ storage.list(
+ bucketName,
+ Storage.BlobListOption.delimiter("/"),
+ Storage.BlobListOption.includeFolders(false));
+
+ boolean found = false;
+ for (Blob blob : blobs.iterateAll()) {
+ if (blob.getName().equals("F/")) {
+ found = true;
+ }
+ }
+ assert (!found);
+
+ blobs =
+ storage.list(
+ bucketName,
+ Storage.BlobListOption.delimiter("/"),
+ Storage.BlobListOption.includeFolders(true));
+
+ for (Blob blob : blobs.iterateAll()) {
+ if (blob.getName().equals("F/")) {
+ found = true;
+ }
+ }
+ assert (found);
+
+ } finally {
+ BucketCleaner.doCleanup(bucketName, storage);
+ }
+ }
 
  private void unsetRequesterPays() {
  Bucket remoteBucket =