Date: Fri, 15 Mar 2024 17:59:27 +0100
Subject: [PATCH 11/14] deps: update dependency net.jqwik:jqwik to v1.8.4
(#2447)
---
google-cloud-storage/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml
index 604b126fb..0004206f4 100644
--- a/google-cloud-storage/pom.xml
+++ b/google-cloud-storage/pom.xml
@@ -248,7 +248,7 @@
net.jqwik
jqwik
- 1.8.3
+ 1.8.4
test
From 12c9db8935f25a5f9e4633af85ea96acaa914d23 Mon Sep 17 00:00:00 2001
From: Sydney Munro <97561403+sydney-munro@users.noreply.github.com>
Date: Fri, 15 Mar 2024 11:13:45 -0700
Subject: [PATCH 12/14] fix: Fix name digest for noprefix (#2448)
---
.../ParallelCompositeUploadBlobWriteSessionConfig.java | 5 ++++-
.../ParallelCompositeUploadBlobWriteSessionConfigTest.java | 2 +-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfig.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfig.java
index b877d82d9..e13f54964 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfig.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfig.java
@@ -612,7 +612,10 @@ public NoPrefix(SecureRandom rand) {
}
@Override
- protected String fmtFields(String randomKey, String nameDigest, String partRange) {
+ protected String fmtFields(String randomKey, String ultimateObjectName, String partRange) {
+ HashCode hashCode =
+ OBJECT_NAME_HASH_FUNCTION.hashString(ultimateObjectName, StandardCharsets.UTF_8);
+ String nameDigest = B64.encodeToString(hashCode.asBytes());
return randomKey
+ FIELD_SEPARATOR
// todo: do we want to
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfigTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfigTest.java
index 86eddc5b9..465741a7d 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfigTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfigTest.java
@@ -84,7 +84,7 @@ public void partNameStrategy_objectNamePrefix() throws Exception {
// name digest
() -> assertField(fmt, 1).hasLength(22),
() -> assertField(fmt, 2).isEqualTo("0001-0096.part"),
- () -> assertThat(fmt).startsWith("a/b/obj/"));
+ () -> assertThat(fmt).startsWith("a/b/obj"));
}
private static StringSubject assertField(String fmt, int idx) {
From 43b800645ba3622e5de635825e1d082c6d26c2eb Mon Sep 17 00:00:00 2001
From: Sydney Munro <97561403+sydney-munro@users.noreply.github.com>
Date: Fri, 15 Mar 2024 11:49:08 -0700
Subject: [PATCH 13/14] feat: Add Custom Part Metadata Decorator to
ParallelCompositeUploadConfig (#2434)
---
...CompositeUploadBlobWriteSessionConfig.java | 139 ++++++++++++++++--
...lelCompositeUploadWritableByteChannel.java | 5 +
...ositeUploadBlobWriteSessionConfigTest.java | 17 +++
...ompositeUploadWritableByteChannelTest.java | 63 +++++++-
.../cloud/storage/SerializationTest.java | 10 +-
.../com/google/cloud/storage/TestClock.java | 2 +-
...ositeUploadBlobWriteSessionConfigTest.java | 19 ++-
7 files changed, 241 insertions(+), 14 deletions(-)
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfig.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfig.java
index e13f54964..07551abe4 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfig.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfig.java
@@ -42,12 +42,15 @@
import java.nio.charset.StandardCharsets;
import java.security.SecureRandom;
import java.time.Clock;
+import java.time.Duration;
+import java.time.OffsetDateTime;
import java.util.Base64;
import java.util.Base64.Encoder;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.UnaryOperator;
import javax.annotation.concurrent.Immutable;
import org.checkerframework.checker.nullness.qual.NonNull;
@@ -125,18 +128,21 @@ public final class ParallelCompositeUploadBlobWriteSessionConfig extends BlobWri
private final BufferAllocationStrategy bufferAllocationStrategy;
private final PartNamingStrategy partNamingStrategy;
private final PartCleanupStrategy partCleanupStrategy;
+ private final PartMetadataFieldDecorator partMetadataFieldDecorator;
private ParallelCompositeUploadBlobWriteSessionConfig(
int maxPartsPerCompose,
ExecutorSupplier executorSupplier,
BufferAllocationStrategy bufferAllocationStrategy,
PartNamingStrategy partNamingStrategy,
- PartCleanupStrategy partCleanupStrategy) {
+ PartCleanupStrategy partCleanupStrategy,
+ PartMetadataFieldDecorator partMetadataFieldDecorator) {
this.maxPartsPerCompose = maxPartsPerCompose;
this.executorSupplier = executorSupplier;
this.bufferAllocationStrategy = bufferAllocationStrategy;
this.partNamingStrategy = partNamingStrategy;
this.partCleanupStrategy = partCleanupStrategy;
+ this.partMetadataFieldDecorator = partMetadataFieldDecorator;
}
@InternalApi
@@ -150,7 +156,8 @@ ParallelCompositeUploadBlobWriteSessionConfig withMaxPartsPerCompose(int maxPart
executorSupplier,
bufferAllocationStrategy,
partNamingStrategy,
- partCleanupStrategy);
+ partCleanupStrategy,
+ partMetadataFieldDecorator);
}
/**
@@ -170,7 +177,8 @@ public ParallelCompositeUploadBlobWriteSessionConfig withExecutorSupplier(
executorSupplier,
bufferAllocationStrategy,
partNamingStrategy,
- partCleanupStrategy);
+ partCleanupStrategy,
+ partMetadataFieldDecorator);
}
/**
@@ -191,7 +199,8 @@ public ParallelCompositeUploadBlobWriteSessionConfig withBufferAllocationStrateg
executorSupplier,
bufferAllocationStrategy,
partNamingStrategy,
- partCleanupStrategy);
+ partCleanupStrategy,
+ partMetadataFieldDecorator);
}
/**
@@ -211,7 +220,8 @@ public ParallelCompositeUploadBlobWriteSessionConfig withPartNamingStrategy(
executorSupplier,
bufferAllocationStrategy,
partNamingStrategy,
- partCleanupStrategy);
+ partCleanupStrategy,
+ partMetadataFieldDecorator);
}
/**
@@ -231,7 +241,29 @@ public ParallelCompositeUploadBlobWriteSessionConfig withPartCleanupStrategy(
executorSupplier,
bufferAllocationStrategy,
partNamingStrategy,
- partCleanupStrategy);
+ partCleanupStrategy,
+ partMetadataFieldDecorator);
+ }
+
+ /**
+ * Specify a Part Metadata Field decorator, this will manipulate the metadata associated with part
+ * objects, the ultimate object metadata will remain unchanged.
+ *
+ * Default: {@link PartMetadataFieldDecorator#noOp()}
+ *
+ * @since 2.36.0 This new api is in preview and is subject to breaking changes.
+ */
+ @BetaApi
+ public ParallelCompositeUploadBlobWriteSessionConfig withPartMetadataFieldDecorator(
+ PartMetadataFieldDecorator partMetadataFieldDecorator) {
+ checkNotNull(partMetadataFieldDecorator, "partMetadataFieldDecorator must be non null");
+ return new ParallelCompositeUploadBlobWriteSessionConfig(
+ maxPartsPerCompose,
+ executorSupplier,
+ bufferAllocationStrategy,
+ partNamingStrategy,
+ partCleanupStrategy,
+ partMetadataFieldDecorator);
}
@BetaApi
@@ -241,7 +273,8 @@ static ParallelCompositeUploadBlobWriteSessionConfig withDefaults() {
ExecutorSupplier.cachedPool(),
BufferAllocationStrategy.simple(ByteSizeConstants._16MiB),
PartNamingStrategy.noPrefix(),
- PartCleanupStrategy.always());
+ PartCleanupStrategy.always(),
+ PartMetadataFieldDecorator.noOp());
}
@InternalApi
@@ -249,7 +282,10 @@ static ParallelCompositeUploadBlobWriteSessionConfig withDefaults() {
WriterFactory createFactory(Clock clock) throws IOException {
Executor executor = executorSupplier.get();
BufferHandlePool bufferHandlePool = bufferAllocationStrategy.get();
- return new ParallelCompositeUploadWriterFactory(clock, executor, bufferHandlePool);
+ PartMetadataFieldDecoratorInstance partMetadataFieldDecoratorInstance =
+ partMetadataFieldDecorator.newInstance(clock);
+ return new ParallelCompositeUploadWriterFactory(
+ clock, executor, bufferHandlePool, partMetadataFieldDecoratorInstance);
}
/**
@@ -277,6 +313,7 @@ private BufferAllocationStrategy() {}
*/
@BetaApi
public static BufferAllocationStrategy simple(int capacity) {
+ checkArgument(capacity > 0, "bufferCapacity must be > 0");
return new SimpleBufferAllocationStrategy(capacity);
}
@@ -291,6 +328,8 @@ public static BufferAllocationStrategy simple(int capacity) {
*/
@BetaApi
public static BufferAllocationStrategy fixedPool(int bufferCount, int bufferCapacity) {
+ checkArgument(bufferCount > 0, "bufferCount must be > 0");
+ checkArgument(bufferCapacity > 0, "bufferCapacity must be > 0");
return new FixedPoolBufferAllocationStrategy(bufferCount, bufferCapacity);
}
@@ -361,6 +400,7 @@ public static ExecutorSupplier cachedPool() {
*/
@BetaApi
public static ExecutorSupplier fixedPool(int poolSize) {
+ checkArgument(poolSize > 0, "poolSize must be > 0");
return new FixedSupplier(poolSize);
}
@@ -631,6 +671,79 @@ protected String fmtFields(String randomKey, String ultimateObjectName, String p
}
}
+ /**
+ * A Decorator which is used to manipulate metadata fields, specifically on the part objects
+ * created in a Parallel Composite Upload
+ *
+ * @see #withPartMetadataFieldDecorator(PartMetadataFieldDecorator)
+ * @since 2.36.0 This new api is in preview and is subject to breaking changes.
+ */
+ @BetaApi
+ @Immutable
+ public abstract static class PartMetadataFieldDecorator implements Serializable {
+
+ abstract PartMetadataFieldDecoratorInstance newInstance(Clock clock);
+
+ /**
+ * A decorator that is used to manipulate the Custom Time Metadata field of part files. {@link
+ * BlobInfo#getCustomTimeOffsetDateTime()}
+ *
+ *
When provided with a duration, a time in the future will be calculated for each part
+ * object upon upload, this new value can be used in OLM rules to cleanup abandoned part files.
+ *
+ *
See [CustomTime OLM
+ * documentation](https://cloud.google.com/storage/docs/lifecycle#dayssincecustomtime)
+ *
+ * @see #withPartMetadataFieldDecorator(PartMetadataFieldDecorator)
+ * @since 2.36.0 This new api is in preview and is subject to breaking changes.
+ */
+ @BetaApi
+ public static PartMetadataFieldDecorator setCustomTimeInFuture(Duration timeInFuture) {
+ checkNotNull(timeInFuture, "timeInFuture must not be null");
+ return new CustomTimeInFuture(timeInFuture);
+ }
+
+ @BetaApi
+ public static PartMetadataFieldDecorator noOp() {
+ return NoOp.INSTANCE;
+ }
+
+ @BetaApi
+ private static final class CustomTimeInFuture extends PartMetadataFieldDecorator {
+ private static final long serialVersionUID = -6213742554954751892L;
+ private final Duration duration;
+
+ CustomTimeInFuture(Duration duration) {
+ this.duration = duration;
+ }
+
+ @Override
+ PartMetadataFieldDecoratorInstance newInstance(Clock clock) {
+ return builder -> {
+ OffsetDateTime futureTime =
+ OffsetDateTime.from(
+ clock.instant().plus(duration).atZone(clock.getZone()).toOffsetDateTime());
+ return builder.setCustomTimeOffsetDateTime(futureTime);
+ };
+ }
+ }
+
+ private static final class NoOp extends PartMetadataFieldDecorator {
+ private static final long serialVersionUID = -4569486383992999205L;
+ private static final NoOp INSTANCE = new NoOp();
+
+ @Override
+ PartMetadataFieldDecoratorInstance newInstance(Clock clock) {
+ return builder -> builder;
+ }
+
+ /** prevent java serialization from using a new instance */
+ private Object readResolve() {
+ return INSTANCE;
+ }
+ }
+ }
+
/**
* A cleanup strategy which will dictate what cleanup operations are performed automatically when
* performing a parallel composite upload.
@@ -708,6 +821,8 @@ public static PartCleanupStrategy never() {
}
}
+ interface PartMetadataFieldDecoratorInstance extends UnaryOperator {}
+
private abstract static class Factory implements Serializable {
private static final long serialVersionUID = 271806144227661056L;
@@ -721,12 +836,17 @@ private class ParallelCompositeUploadWriterFactory implements WriterFactory {
private final Clock clock;
private final Executor executor;
private final BufferHandlePool bufferHandlePool;
+ private final PartMetadataFieldDecoratorInstance partMetadataFieldDecoratorInstance;
private ParallelCompositeUploadWriterFactory(
- Clock clock, Executor executor, BufferHandlePool bufferHandlePool) {
+ Clock clock,
+ Executor executor,
+ BufferHandlePool bufferHandlePool,
+ PartMetadataFieldDecoratorInstance partMetadataFieldDecoratorInstance) {
this.clock = clock;
this.executor = executor;
this.bufferHandlePool = bufferHandlePool;
+ this.partMetadataFieldDecoratorInstance = partMetadataFieldDecoratorInstance;
}
@Override
@@ -760,6 +880,7 @@ public ApiFuture openAsync() {
partNamingStrategy,
partCleanupStrategy,
maxPartsPerCompose,
+ partMetadataFieldDecoratorInstance,
result,
storageInternal,
info,
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ParallelCompositeUploadWritableByteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ParallelCompositeUploadWritableByteChannel.java
index 9ff1ebdb5..639802cde 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/ParallelCompositeUploadWritableByteChannel.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ParallelCompositeUploadWritableByteChannel.java
@@ -31,6 +31,7 @@
import com.google.cloud.storage.BufferedWritableByteChannelSession.BufferedWritableByteChannel;
import com.google.cloud.storage.MetadataField.PartRange;
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartCleanupStrategy;
+import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartMetadataFieldDecoratorInstance;
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartNamingStrategy;
import com.google.cloud.storage.Storage.ComposeRequest;
import com.google.cloud.storage.UnifiedOpts.Crc32cMatch;
@@ -111,6 +112,7 @@ final class ParallelCompositeUploadWritableByteChannel implements BufferedWritab
private final PartNamingStrategy partNamingStrategy;
private final PartCleanupStrategy partCleanupStrategy;
private final int maxElementsPerCompact;
+ private final PartMetadataFieldDecoratorInstance partMetadataFieldDecorator;
private final SettableApiFuture finalObject;
private final StorageInternal storage;
private final BlobInfo ultimateObject;
@@ -135,6 +137,7 @@ final class ParallelCompositeUploadWritableByteChannel implements BufferedWritab
PartNamingStrategy partNamingStrategy,
PartCleanupStrategy partCleanupStrategy,
int maxElementsPerCompact,
+ PartMetadataFieldDecoratorInstance partMetadataFieldDecorator,
SettableApiFuture finalObject,
StorageInternal storage,
BlobInfo ultimateObject,
@@ -144,6 +147,7 @@ final class ParallelCompositeUploadWritableByteChannel implements BufferedWritab
this.partNamingStrategy = partNamingStrategy;
this.partCleanupStrategy = partCleanupStrategy;
this.maxElementsPerCompact = maxElementsPerCompact;
+ this.partMetadataFieldDecorator = partMetadataFieldDecorator;
this.finalObject = finalObject;
this.storage = storage;
this.ultimateObject = ultimateObject;
@@ -427,6 +431,7 @@ private BlobInfo definePart(BlobInfo ultimateObject, PartRange partRange, long o
PART_INDEX.appendTo(partRange, builder);
OBJECT_OFFSET.appendTo(offset, builder);
b.setMetadata(builder.build());
+ b = partMetadataFieldDecorator.apply(b);
return b.build();
}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfigTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfigTest.java
index 465741a7d..3ef553727 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfigTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfigTest.java
@@ -21,8 +21,13 @@
import static com.google.common.truth.Truth.assertWithMessage;
import com.google.cloud.storage.MetadataField.PartRange;
+import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartMetadataFieldDecorator;
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartNamingStrategy;
import com.google.common.truth.StringSubject;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
import org.junit.Test;
public final class ParallelCompositeUploadBlobWriteSessionConfigTest {
@@ -87,6 +92,18 @@ public void partNameStrategy_objectNamePrefix() throws Exception {
() -> assertThat(fmt).startsWith("a/b/obj"));
}
+ @Test
+ public void partMetadataFieldDecorator_customTime() {
+ BlobInfo.Builder testBlob = BlobInfo.newBuilder("testBlob", "testBucket");
+ Duration duration = Duration.ofSeconds(30);
+ TestClock clock = TestClock.tickBy(Instant.EPOCH, Duration.ofSeconds(1));
+ OffsetDateTime expected =
+ OffsetDateTime.from(Instant.EPOCH.plus(duration).atZone(ZoneId.of("Z")));
+ PartMetadataFieldDecorator.setCustomTimeInFuture(duration).newInstance(clock).apply(testBlob);
+
+ assertThat(expected).isEqualTo(testBlob.build().getCustomTimeOffsetDateTime());
+ }
+
private static StringSubject assertField(String fmt, int idx) {
String[] split = fmt.split(";");
String s = split[idx];
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadWritableByteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadWritableByteChannelTest.java
index ed66a3f17..3a78adf4f 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadWritableByteChannelTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadWritableByteChannelTest.java
@@ -29,6 +29,8 @@
import com.google.cloud.storage.Crc32cValue.Crc32cLengthKnown;
import com.google.cloud.storage.MetadataField.PartRange;
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartCleanupStrategy;
+import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartMetadataFieldDecorator;
+import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartMetadataFieldDecoratorInstance;
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartNamingStrategy;
import com.google.cloud.storage.ParallelCompositeUploadWritableByteChannel.BufferHandleReleaser;
import com.google.cloud.storage.Storage.ComposeRequest;
@@ -47,9 +49,14 @@
import com.google.common.util.concurrent.Uninterruptibles;
import com.google.storage.v2.WriteObjectRequest;
import io.grpc.Status.Code;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.AsynchronousCloseException;
import java.nio.channels.ClosedChannelException;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collections;
@@ -80,6 +87,7 @@ public final class ParallelCompositeUploadWritableByteChannelTest {
private SettableApiFuture finalObject;
private FakeStorageInternal storageInternal;
private SimplisticPartNamingStrategy partNamingStrategy;
+ private PartMetadataFieldDecoratorInstance partMetadataFieldDecorator;
private int bufferCapacity;
@Before
@@ -91,6 +99,7 @@ public void setUp() throws Exception {
finalObject = SettableApiFuture.create();
partNamingStrategy = new SimplisticPartNamingStrategy("prefix");
storageInternal = new FakeStorageInternal();
+ partMetadataFieldDecorator = PartMetadataFieldDecorator.noOp().newInstance(null);
}
@Test
@@ -202,6 +211,7 @@ public void cleanup_success_disabled() throws Exception {
partNamingStrategy,
PartCleanupStrategy.never(),
maxElementsPerCompact,
+ partMetadataFieldDecorator,
finalObject,
storageInternal,
info,
@@ -241,6 +251,7 @@ public void writeDoesNotFlushIfItIsnNotFull() throws Exception {
partNamingStrategy,
PartCleanupStrategy.never(),
maxElementsPerCompact,
+ partMetadataFieldDecorator,
finalObject,
storageInternal,
info,
@@ -340,6 +351,7 @@ public void partsRetainMetadata() throws Exception {
partNamingStrategy,
PartCleanupStrategy.never(),
3,
+ partMetadataFieldDecorator,
finalObject,
new FakeStorageInternal() {
@Override
@@ -429,6 +441,7 @@ public void creatingAnEmptyObjectWhichFailsIsSetAsResultFailureAndThrowFromClose
partNamingStrategy,
PartCleanupStrategy.always(),
3,
+ partMetadataFieldDecorator,
finalObject,
new FakeStorageInternal() {
@Override
@@ -462,6 +475,7 @@ public void badServerCrc32cResultsInException() throws Exception {
partNamingStrategy,
PartCleanupStrategy.always(),
3,
+ partMetadataFieldDecorator,
finalObject,
new FakeStorageInternal() {
@Override
@@ -568,6 +582,7 @@ public BlobInfo internalDirectUpload(
partNamingStrategy,
PartCleanupStrategy.never(),
32,
+ partMetadataFieldDecorator,
finalObject,
storageInternal,
info,
@@ -647,6 +662,7 @@ public void errorContextIsPopulated() throws Exception {
partNamingStrategy,
PartCleanupStrategy.never(),
3,
+ partMetadataFieldDecorator,
finalObject,
new FakeStorageInternal() {
@Override
@@ -729,6 +745,7 @@ public BlobInfo internalObjectGet(BlobId blobId, Opts opts) {
partNamingStrategy,
PartCleanupStrategy.always(),
10,
+ partMetadataFieldDecorator,
finalObject,
storageInternal,
info,
@@ -749,6 +766,49 @@ public BlobInfo internalObjectGet(BlobId blobId, Opts opts) {
() -> assertThat(storageInternal.deleteRequests).containsExactly(p1, p2, p3));
}
+ @Test
+ public void partMetadataFieldDecoratorUsesCustomTime() throws IOException {
+ TestClock clock = TestClock.tickBy(Instant.EPOCH, Duration.ofSeconds(1));
+ OffsetDateTime rangeBegin =
+ OffsetDateTime.from(Instant.EPOCH.plus(Duration.ofSeconds(29)).atZone(ZoneId.of("Z")));
+ OffsetDateTime rangeEnd =
+ OffsetDateTime.from(Instant.EPOCH.plus(Duration.ofMinutes(2)).atZone(ZoneId.of("Z")));
+
+ FakeStorageInternal storageInternal =
+ new FakeStorageInternal() {
+ @Override
+ public BlobInfo internalDirectUpload(
+ BlobInfo info, Opts opts, ByteBuffer buf) {
+ if (info.getBlobId().getName().endsWith(".part")) {
+ // Kinda hacky but since we are creating multiple parts we will use a range
+ // to ensure the customTimes are being calculated appropriately
+ assertThat(info.getCustomTimeOffsetDateTime().isAfter(rangeBegin)).isTrue();
+ assertThat(info.getCustomTimeOffsetDateTime().isBefore(rangeEnd)).isTrue();
+ } else {
+ assertThat(info.getCustomTimeOffsetDateTime()).isNull();
+ }
+ return super.internalDirectUpload(info, opts, buf);
+ }
+ };
+ ParallelCompositeUploadWritableByteChannel pcu =
+ new ParallelCompositeUploadWritableByteChannel(
+ bufferHandlePool,
+ MoreExecutors.directExecutor(),
+ partNamingStrategy,
+ PartCleanupStrategy.always(),
+ 10,
+ PartMetadataFieldDecorator.setCustomTimeInFuture(Duration.ofSeconds(30))
+ .newInstance(clock),
+ finalObject,
+ storageInternal,
+ info,
+ opts);
+ byte[] bytes = DataGenerator.base64Characters().genBytes(bufferCapacity * 3 - 1);
+ pcu.write(ByteBuffer.wrap(bytes));
+
+ pcu.close();
+ }
+
@NonNull
private ParallelCompositeUploadWritableByteChannel defaultPcu(int maxElementsPerCompact) {
return new ParallelCompositeUploadWritableByteChannel(
@@ -757,6 +817,7 @@ private ParallelCompositeUploadWritableByteChannel defaultPcu(int maxElementsPer
partNamingStrategy,
PartCleanupStrategy.always(),
maxElementsPerCompact,
+ partMetadataFieldDecorator,
finalObject,
storageInternal,
info,
@@ -773,7 +834,7 @@ private static class FakeStorageInternal implements StorageInternal {
protected final List composeRequests;
protected final List deleteRequests;
- private FakeStorageInternal() {
+ FakeStorageInternal() {
generations = new AtomicInteger(1);
addedObjects = Collections.synchronizedMap(new HashMap<>());
composeRequests = Collections.synchronizedList(new ArrayList<>());
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/SerializationTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/SerializationTest.java
index eab0e78ed..5f642b50b 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/SerializationTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/SerializationTest.java
@@ -36,6 +36,7 @@
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.BufferAllocationStrategy;
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.ExecutorSupplier;
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartCleanupStrategy;
+import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartMetadataFieldDecorator;
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartNamingStrategy;
import com.google.cloud.storage.Storage.BlobTargetOption;
import com.google.cloud.storage.Storage.BucketField;
@@ -55,6 +56,7 @@
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
+import java.time.Duration;
import java.util.Base64;
import java.util.Collections;
import java.util.Map;
@@ -388,10 +390,16 @@ public void blobWriteSessionConfig_pcu() throws IOException, ClassNotFoundExcept
.withBufferAllocationStrategy(BufferAllocationStrategy.fixedPool(1, 3))
.withPartCleanupStrategy(PartCleanupStrategy.never())
.withPartNamingStrategy(PartNamingStrategy.prefix("prefix"))
- .withExecutorSupplier(ExecutorSupplier.fixedPool(5));
+ .withExecutorSupplier(ExecutorSupplier.fixedPool(5))
+ .withPartMetadataFieldDecorator(
+ PartMetadataFieldDecorator.setCustomTimeInFuture(Duration.ofMinutes(10)));
ParallelCompositeUploadBlobWriteSessionConfig pcu2copy = serializeAndDeserialize(pcu2);
assertThat(pcu2copy).isNotNull();
+ PartMetadataFieldDecorator noop = PartMetadataFieldDecorator.noOp();
+ PartMetadataFieldDecorator noopCopy = serializeAndDeserialize(noop);
+ assertThat(noopCopy).isSameInstanceAs(noop);
+
InvalidClassException invalidClassException =
assertThrows(
InvalidClassException.class,
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/TestClock.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/TestClock.java
index e37a4f024..89d04c35c 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/TestClock.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/TestClock.java
@@ -37,7 +37,7 @@ private TestClock(Instant begin, UnaryOperator next) {
@Override
public ZoneId getZone() {
- throw new UnsupportedOperationException("TestClock.getZone()");
+ return ZoneId.of("Z");
}
@Override
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITParallelCompositeUploadBlobWriteSessionConfigTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITParallelCompositeUploadBlobWriteSessionConfigTest.java
index 012a700f6..304cdd69f 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITParallelCompositeUploadBlobWriteSessionConfigTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITParallelCompositeUploadBlobWriteSessionConfigTest.java
@@ -21,6 +21,7 @@
import static com.google.common.truth.Truth.assertWithMessage;
import static org.junit.Assert.assertThrows;
+import com.google.api.gax.paging.Page;
import com.google.api.gax.rpc.ApiExceptions;
import com.google.cloud.kms.v1.CryptoKey;
import com.google.cloud.storage.Blob;
@@ -34,6 +35,7 @@
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.BufferAllocationStrategy;
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.ExecutorSupplier;
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartCleanupStrategy;
+import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartMetadataFieldDecorator;
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartNamingStrategy;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.Storage.BlobSourceOption;
@@ -57,6 +59,7 @@
import java.nio.channels.AsynchronousCloseException;
import java.nio.channels.WritableByteChannel;
import java.security.Key;
+import java.time.Duration;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -109,8 +112,11 @@ public void setUp() throws Exception {
// define a max part size that is fairly small to aid in test speed
.withBufferAllocationStrategy(BufferAllocationStrategy.simple(_1MiB))
.withPartNamingStrategy(PartNamingStrategy.prefix("prefix-a"))
- // let our fixtures take care of cleaning things up if an upload fails
- .withPartCleanupStrategy(PartCleanupStrategy.onlyOnSuccess());
+ // Write customTime 30 seconds in the future
+ .withPartMetadataFieldDecorator(
+ PartMetadataFieldDecorator.setCustomTimeInFuture(Duration.ofSeconds(30)))
+ // let our fixtures take care of cleaning things
+ .withPartCleanupStrategy(PartCleanupStrategy.never());
StorageOptions storageOptions = null;
if (transport == Transport.GRPC) {
@@ -140,6 +146,15 @@ public static void afterClass() {
}
}
+ @Test
+ public void partFilesCreatedWithCustomTimeWritten() throws IOException {
+ doTest(bucket, 10 * _1MiB + 37, ImmutableList.of(), ImmutableList.of(), ImmutableList.of());
+ Page blobs = storage.list(bucket.getName(), Storage.BlobListOption.prefix("prefix-a"));
+ for (Blob blob : blobs.iterateAll()) {
+ assertThat(blob.getCustomTimeOffsetDateTime()).isNotNull();
+ }
+ }
+
@Test
public void errorRaisedByMethodAndFutureResult() throws IOException {
From 838f9f86edb674aa3352ba2f6c6011bc2f5c5ea7 Mon Sep 17 00:00:00 2001
From: "release-please[bot]"
<55107282+release-please[bot]@users.noreply.github.com>
Date: Fri, 15 Mar 2024 14:29:17 -0700
Subject: [PATCH 14/14] chore(main): release 2.36.0 (#2443)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* chore(main): release 2.36.0
* 🦉 Updates from OwlBot post-processor
See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md
---------
Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com>
Co-authored-by: Owl Bot
---
CHANGELOG.md | 22 +++++++++++++++++++
gapic-google-cloud-storage-v2/pom.xml | 4 ++--
google-cloud-storage-bom/pom.xml | 16 +++++++-------
google-cloud-storage-control/pom.xml | 4 ++--
google-cloud-storage/pom.xml | 4 ++--
grpc-google-cloud-storage-control-v2/pom.xml | 4 ++--
grpc-google-cloud-storage-v2/pom.xml | 4 ++--
pom.xml | 16 +++++++-------
proto-google-cloud-storage-control-v2/pom.xml | 4 ++--
proto-google-cloud-storage-v2/pom.xml | 4 ++--
samples/snapshot/pom.xml | 2 +-
storage-shared-benchmarking/pom.xml | 4 ++--
versions.txt | 14 ++++++------
13 files changed, 62 insertions(+), 40 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bac6556a3..e2eb26d4d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,27 @@
# Changelog
+## [2.36.0](https://github.com/googleapis/java-storage/compare/v2.35.0...v2.36.0) (2024-03-15)
+
+
+### Features
+
+* Add Custom Part Metadata Decorator to ParallelCompositeUploadConfig ([#2434](https://github.com/googleapis/java-storage/issues/2434)) ([43b8006](https://github.com/googleapis/java-storage/commit/43b800645ba3622e5de635825e1d082c6d26c2eb))
+* Add hierarchical namespace and folders features ([#2445](https://github.com/googleapis/java-storage/issues/2445)) ([8074fff](https://github.com/googleapis/java-storage/commit/8074fffed5208a8578e5afe694fdd3d8df627b8c))
+* Add soft delete feature ([#2403](https://github.com/googleapis/java-storage/issues/2403)) ([989f36f](https://github.com/googleapis/java-storage/commit/989f36fbb206832a6a3584c77546d3d560ac0df8))
+
+
+### Bug Fixes
+
+* Fix name digest for noprefix ([#2448](https://github.com/googleapis/java-storage/issues/2448)) ([12c9db8](https://github.com/googleapis/java-storage/commit/12c9db8935f25a5f9e4633af85ea96acaa914d23))
+* Missing serialVersionUID of serializable classes ([#2344](https://github.com/googleapis/java-storage/issues/2344)) ([736865b](https://github.com/googleapis/java-storage/commit/736865b4b97aa3940e1eab6a582d0ef38db31bba))
+
+
+### Dependencies
+
+* Update dependency com.google.apis:google-api-services-storage to v1-rev20240307-2.0.0 ([#2442](https://github.com/googleapis/java-storage/issues/2442)) ([1352203](https://github.com/googleapis/java-storage/commit/1352203859c3798423ef78823ed10577b93eebef))
+* Update dependency net.jqwik:jqwik to v1.8.4 ([#2447](https://github.com/googleapis/java-storage/issues/2447)) ([110b80c](https://github.com/googleapis/java-storage/commit/110b80cdde24da4868e46c3909020db7b02d5491))
+* Update gcr.io/cloud-devrel-public-resources/storage-testbench docker tag to v0.42.0 ([#2441](https://github.com/googleapis/java-storage/issues/2441)) ([80745d4](https://github.com/googleapis/java-storage/commit/80745d415810353b3466a3eda12f6ca50a2c71b0))
+
## [2.35.0](https://github.com/googleapis/java-storage/compare/v2.34.0...v2.35.0) (2024-03-04)
diff --git a/gapic-google-cloud-storage-v2/pom.xml b/gapic-google-cloud-storage-v2/pom.xml
index d02b72dd1..5e15e2a90 100644
--- a/gapic-google-cloud-storage-v2/pom.xml
+++ b/gapic-google-cloud-storage-v2/pom.xml
@@ -4,13 +4,13 @@
4.0.0
com.google.api.grpc
gapic-google-cloud-storage-v2
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
gapic-google-cloud-storage-v2
GRPC library for gapic-google-cloud-storage-v2
com.google.cloud
google-cloud-storage-parent
- 2.35.1-SNAPSHOT
+ 2.36.0
diff --git a/google-cloud-storage-bom/pom.xml b/google-cloud-storage-bom/pom.xml
index bc0311201..f0489d5a3 100644
--- a/google-cloud-storage-bom/pom.xml
+++ b/google-cloud-storage-bom/pom.xml
@@ -19,7 +19,7 @@
4.0.0
com.google.cloud
google-cloud-storage-bom
- 2.35.1-SNAPSHOT
+ 2.36.0
pom
com.google.cloud
@@ -69,37 +69,37 @@
com.google.cloud
google-cloud-storage
- 2.35.1-SNAPSHOT
+ 2.36.0
com.google.api.grpc
gapic-google-cloud-storage-v2
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
com.google.api.grpc
grpc-google-cloud-storage-v2
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
com.google.api.grpc
proto-google-cloud-storage-v2
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
com.google.cloud
google-cloud-storage-control
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
com.google.api.grpc
grpc-google-cloud-storage-control-v2
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
com.google.api.grpc
proto-google-cloud-storage-control-v2
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
diff --git a/google-cloud-storage-control/pom.xml b/google-cloud-storage-control/pom.xml
index 747b2b3d7..7663e73a6 100644
--- a/google-cloud-storage-control/pom.xml
+++ b/google-cloud-storage-control/pom.xml
@@ -5,13 +5,13 @@
4.0.0
com.google.cloud
google-cloud-storage-control
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
google-cloud-storage-control
GRPC library for google-cloud-storage-control
com.google.cloud
google-cloud-storage-parent
- 2.35.1-SNAPSHOT
+ 2.36.0
diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml
index 0004206f4..35f4cd2b0 100644
--- a/google-cloud-storage/pom.xml
+++ b/google-cloud-storage/pom.xml
@@ -2,7 +2,7 @@
4.0.0
google-cloud-storage
- 2.35.1-SNAPSHOT
+ 2.36.0
jar
Google Cloud Storage
https://github.com/googleapis/java-storage
@@ -12,7 +12,7 @@
com.google.cloud
google-cloud-storage-parent
- 2.35.1-SNAPSHOT
+ 2.36.0
google-cloud-storage
diff --git a/grpc-google-cloud-storage-control-v2/pom.xml b/grpc-google-cloud-storage-control-v2/pom.xml
index 1b9b70b4e..b3813b522 100644
--- a/grpc-google-cloud-storage-control-v2/pom.xml
+++ b/grpc-google-cloud-storage-control-v2/pom.xml
@@ -4,13 +4,13 @@
4.0.0
com.google.api.grpc
grpc-google-cloud-storage-control-v2
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
grpc-google-cloud-storage-control-v2
GRPC library for google-cloud-storage
com.google.cloud
google-cloud-storage-parent
- 2.35.1-SNAPSHOT
+ 2.36.0
diff --git a/grpc-google-cloud-storage-v2/pom.xml b/grpc-google-cloud-storage-v2/pom.xml
index a8f618376..07b946fef 100644
--- a/grpc-google-cloud-storage-v2/pom.xml
+++ b/grpc-google-cloud-storage-v2/pom.xml
@@ -4,13 +4,13 @@
4.0.0
com.google.api.grpc
grpc-google-cloud-storage-v2
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
grpc-google-cloud-storage-v2
GRPC library for grpc-google-cloud-storage-v2
com.google.cloud
google-cloud-storage-parent
- 2.35.1-SNAPSHOT
+ 2.36.0
diff --git a/pom.xml b/pom.xml
index 98845612a..b442bca53 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
com.google.cloud
google-cloud-storage-parent
pom
- 2.35.1-SNAPSHOT
+ 2.36.0
Storage Parent
https://github.com/googleapis/java-storage
@@ -76,7 +76,7 @@
com.google.cloud
google-cloud-storage
- 2.35.1-SNAPSHOT
+ 2.36.0
com.google.apis
@@ -117,32 +117,32 @@
com.google.api.grpc
proto-google-cloud-storage-v2
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
com.google.api.grpc
grpc-google-cloud-storage-v2
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
com.google.api.grpc
gapic-google-cloud-storage-v2
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
com.google.api.grpc
grpc-google-cloud-storage-control-v2
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
com.google.api.grpc
proto-google-cloud-storage-control-v2
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
com.google.cloud
google-cloud-storage-control
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
com.google.cloud
diff --git a/proto-google-cloud-storage-control-v2/pom.xml b/proto-google-cloud-storage-control-v2/pom.xml
index f028d71dd..9eb168b6e 100644
--- a/proto-google-cloud-storage-control-v2/pom.xml
+++ b/proto-google-cloud-storage-control-v2/pom.xml
@@ -4,13 +4,13 @@
4.0.0
com.google.api.grpc
proto-google-cloud-storage-control-v2
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
proto-google-cloud-storage-control-v2
Proto library for proto-google-cloud-storage-control-v2
com.google.cloud
google-cloud-storage-parent
- 2.35.1-SNAPSHOT
+ 2.36.0
diff --git a/proto-google-cloud-storage-v2/pom.xml b/proto-google-cloud-storage-v2/pom.xml
index 6dcaf9449..dab19adc0 100644
--- a/proto-google-cloud-storage-v2/pom.xml
+++ b/proto-google-cloud-storage-v2/pom.xml
@@ -4,13 +4,13 @@
4.0.0
com.google.api.grpc
proto-google-cloud-storage-v2
- 2.35.1-alpha-SNAPSHOT
+ 2.36.0-alpha
proto-google-cloud-storage-v2
PROTO library for proto-google-cloud-storage-v2
com.google.cloud
google-cloud-storage-parent
- 2.35.1-SNAPSHOT
+ 2.36.0
diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml
index 6211bd424..67c0b685d 100644
--- a/samples/snapshot/pom.xml
+++ b/samples/snapshot/pom.xml
@@ -28,7 +28,7 @@
com.google.cloud
google-cloud-storage
- 2.35.1-SNAPSHOT
+ 2.36.0
diff --git a/storage-shared-benchmarking/pom.xml b/storage-shared-benchmarking/pom.xml
index 5bb630203..5ede2f48a 100644
--- a/storage-shared-benchmarking/pom.xml
+++ b/storage-shared-benchmarking/pom.xml
@@ -10,7 +10,7 @@
com.google.cloud
google-cloud-storage-parent
- 2.35.1-SNAPSHOT
+ 2.36.0
@@ -31,7 +31,7 @@
com.google.cloud
google-cloud-storage
- 2.35.1-SNAPSHOT
+ 2.36.0
tests
diff --git a/versions.txt b/versions.txt
index 1ed68144a..41866bb30 100644
--- a/versions.txt
+++ b/versions.txt
@@ -1,10 +1,10 @@
# Format:
# module:released-version:current-version
-google-cloud-storage:2.35.0:2.35.1-SNAPSHOT
-gapic-google-cloud-storage-v2:2.35.0-alpha:2.35.1-alpha-SNAPSHOT
-grpc-google-cloud-storage-v2:2.35.0-alpha:2.35.1-alpha-SNAPSHOT
-proto-google-cloud-storage-v2:2.35.0-alpha:2.35.1-alpha-SNAPSHOT
-google-cloud-storage-control:2.35.0-alpha:2.35.1-alpha-SNAPSHOT
-proto-google-cloud-storage-control-v2:2.35.0-alpha:2.35.1-alpha-SNAPSHOT
-grpc-google-cloud-storage-control-v2:2.35.0-alpha:2.35.1-alpha-SNAPSHOT
+google-cloud-storage:2.36.0:2.36.0
+gapic-google-cloud-storage-v2:2.36.0-alpha:2.36.0-alpha
+grpc-google-cloud-storage-v2:2.36.0-alpha:2.36.0-alpha
+proto-google-cloud-storage-v2:2.36.0-alpha:2.36.0-alpha
+google-cloud-storage-control:2.36.0-alpha:2.36.0-alpha
+proto-google-cloud-storage-control-v2:2.36.0-alpha:2.36.0-alpha
+grpc-google-cloud-storage-control-v2:2.36.0-alpha:2.36.0-alpha