Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Add Custom Part Metadata Decorator to ParallelCompositeUploadConfig #2434

Merged
merged 24 commits into from
Mar 15, 2024
Merged

feat: Add Custom Part Metadata Decorator to ParallelCompositeUploadConfig #2434

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
33122ad
feat: Custom Time Part Metadata Strategy
sydney-munro Mar 4, 2024
fe168d6
set customtime directly instead of via metadata builder
sydney-munro Mar 4, 2024
d14d141
lint and change null to IllegalStateException
sydney-munro Mar 4, 2024
9f12759
Add in version to javadoc
sydney-munro Mar 5, 2024
71ec5cf
removing unnecessary override
sydney-munro Mar 5, 2024
8b64aa6
removing unnecessary tag
sydney-munro Mar 5, 2024
15a29dc
address pr comments
sydney-munro Mar 13, 2024
b1bc015
add in ability to set a specific clock
sydney-munro Mar 13, 2024
19297a0
chore: refactor and separate lifecycle of config from customer and wh…
BenWhitehead Mar 13, 2024
4a51be1
chore: add pool size validation
BenWhitehead Mar 13, 2024
2a98f6c
IT test to ensure customtime is updated
sydney-munro Mar 14, 2024
746b581
lint
sydney-munro Mar 14, 2024
4acfa8c
Adding in more tests
sydney-munro Mar 14, 2024
f4e7fa8
lint
sydney-munro Mar 14, 2024
e88574b
more tests
sydney-munro Mar 14, 2024
57597dd
lint
sydney-munro Mar 14, 2024
80c747d
add missing null check
sydney-munro Mar 14, 2024
2adc0e5
address pr comments
sydney-munro Mar 14, 2024
6678349
unit test for offset calculations
sydney-munro Mar 15, 2024
7d39672
fixing docs
sydney-munro Mar 15, 2024
f7c4358
lint
sydney-munro Mar 15, 2024
6762593
Update google-cloud-storage/src/main/java/com/google/cloud/storage/Pa…
sydney-munro Mar 15, 2024
4f51449
Apply suggestions from code review
sydney-munro Mar 15, 2024
4861c85
lint
sydney-munro Mar 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Prev Previous commit
Next Next commit
lint and change null to IllegalStateException
  • Loading branch information
@sydney-munro
sydney-munro committed Mar 15, 2024
commit d14d141bf5ddf05b84910e890d8dea924c2304d7
29 changes: 18 additions & 11 deletions ...src/main/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,8 @@ private ParallelCompositeUploadBlobWriteSessionConfig(
ExecutorSupplier executorSupplier,
BufferAllocationStrategy bufferAllocationStrategy,
PartNamingStrategy partNamingStrategy,
PartCleanupStrategy partCleanupStrategy,PartCustomTimeStrategy partCustomTimeStrategy) {
PartCleanupStrategy partCleanupStrategy,
PartCustomTimeStrategy partCustomTimeStrategy) {
this.maxPartsPerCompose = maxPartsPerCompose;
this.executorSupplier = executorSupplier;
this.bufferAllocationStrategy = bufferAllocationStrategy;
Expand Down Expand Up @@ -243,9 +244,11 @@ public ParallelCompositeUploadBlobWriteSessionConfig withPartCleanupStrategy(
}

@BetaApi
BenWhitehead marked this conversation as resolved.
Show resolved Hide resolved
public ParallelCompositeUploadBlobWriteSessionConfig withPartCustomTimeStrategy(PartCustomTimeStrategy partCustomTimeStrategy) {
public ParallelCompositeUploadBlobWriteSessionConfig withPartCustomTimeStrategy(
PartCustomTimeStrategy partCustomTimeStrategy) {
checkNotNull(partCustomTimeStrategy, "partCustomTimeStrategy must be non null");
return new ParallelCompositeUploadBlobWriteSessionConfig(maxPartsPerCompose,
return new ParallelCompositeUploadBlobWriteSessionConfig(
maxPartsPerCompose,
executorSupplier,
bufferAllocationStrategy,
partNamingStrategy,
Expand Down Expand Up @@ -651,10 +654,9 @@ protected String fmtFields(String randomKey, String ultimateObjectName, String p
}
}
/**
* A strategy which will be used to generate a value for a part or intermediary compose
* object's CustomTime Metadata Field. This will be a time set a duration in the future
* which will serve to aid in part cleanup via OLM Rules.
*
* A strategy which will be used to generate a value for a part or intermediary compose object's
* CustomTime Metadata Field. This will be a time set a duration in the future which will serve to
* aid in part cleanup via OLM Rules.
*
* @see #withPartCustomTimeStrategy(PartCustomTimeStrategy)
* @since <TBD></> This new api is in preview and is subject to breaking changes.
Expand All @@ -667,7 +669,8 @@ public abstract static class PartCustomTimeStrategy implements Serializable {
private PartCustomTimeStrategy(boolean isSetCustomTime) {
this.isSetCustomTime = isSetCustomTime;
}
abstract Duration getTimeInFuture();

abstract Duration getTimeInFuture();

public boolean isSetCustomTime() {
return isSetCustomTime;
Expand All @@ -677,30 +680,33 @@ public boolean isSetCustomTime() {
public static CustomTimeSet setCustomTime(Duration timeInFuture) {
return new CustomTimeSet(timeInFuture);
}

@BetaApi
public static NoCustomTime noCustomTime() {
return new NoCustomTime();
}


static final class CustomTimeSet extends PartCustomTimeStrategy {
private final Duration timeInFuture;

protected Duration getTimeInFuture() {
return timeInFuture;
}

CustomTimeSet(Duration timeInFuture) {
super(true);
this.timeInFuture = timeInFuture;
}
}

static final class NoCustomTime extends PartCustomTimeStrategy {
NoCustomTime() {
super(false);
}

@Override
Duration getTimeInFuture() {
return null;
throw new IllegalStateException("There is no time in future for NoCustomTime Strategy");
}
}
}
Expand Down Expand Up @@ -834,7 +840,8 @@ public ApiFuture<BufferedWritableByteChannel> openAsync() {
partNamingStrategy,
partCleanupStrategy,
maxPartsPerCompose,
partCustomTimeStrategy, result,
partCustomTimeStrategy,
result,
storageInternal,
info,
opts);
Expand Down
5 changes: 3 additions & 2 deletions ...ge/src/main/java/com/google/cloud/storage/ParallelCompositeUploadWritableByteChannel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,8 @@ final class ParallelCompositeUploadWritableByteChannel implements BufferedWritab
PartNamingStrategy partNamingStrategy,
PartCleanupStrategy partCleanupStrategy,
int maxElementsPerCompact,
PartCustomTimeStrategy partCustomTimeStrategy, SettableApiFuture<BlobInfo> finalObject,
PartCustomTimeStrategy partCustomTimeStrategy,
SettableApiFuture<BlobInfo> finalObject,
StorageInternal storage,
BlobInfo ultimateObject,
Opts<ObjectTargetOpt> opts) {
Expand Down Expand Up @@ -432,7 +433,7 @@ private BlobInfo definePart(BlobInfo ultimateObject, PartRange partRange, long o
PART_INDEX.appendTo(partRange, builder);
OBJECT_OFFSET.appendTo(offset, builder);
b.setMetadata(builder.build());
if(partCustomTimeStrategy.isSetCustomTime()) {
if (partCustomTimeStrategy.isSetCustomTime()) {
Duration timeInFuture = partCustomTimeStrategy.getTimeInFuture();
OffsetDateTime now = OffsetDateTime.now();
b.setCustomTimeOffsetDateTime(now.plus(timeInFuture));
Expand Down
3 changes: 2 additions & 1 deletion ...test/java/com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfigTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,9 @@ public void partCustomTimeStrategy_noCustomTime() {
assertThat(strategy.isSetCustomTime()).isFalse();
assertThat(strategy.getTimeInFuture()).isNull();
}

@Test
public void partCustomTimeStrategy_customTimeSet(){
public void partCustomTimeStrategy_customTimeSet() {
Duration timeInFuture = Duration.ofSeconds(30);
PartCustomTimeStrategy strategy = PartCustomTimeStrategy.setCustomTime(timeInFuture);
assertThat(strategy.isSetCustomTime()).isTrue();
Expand Down
27 changes: 18 additions & 9 deletions ...rc/test/java/com/google/cloud/storage/ParallelCompositeUploadWritableByteChannelTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,8 @@ public void cleanup_success_disabled() throws Exception {
partNamingStrategy,
PartCleanupStrategy.never(),
maxElementsPerCompact,
partCustomTimeStrategy, finalObject,
partCustomTimeStrategy,
finalObject,
storageInternal,
info,
opts);
Expand Down Expand Up @@ -244,7 +245,8 @@ public void writeDoesNotFlushIfItIsnNotFull() throws Exception {
partNamingStrategy,
PartCleanupStrategy.never(),
maxElementsPerCompact,
partCustomTimeStrategy, finalObject,
partCustomTimeStrategy,
finalObject,
storageInternal,
info,
opts);
Expand Down Expand Up @@ -343,7 +345,8 @@ public void partsRetainMetadata() throws Exception {
partNamingStrategy,
PartCleanupStrategy.never(),
3,
partCustomTimeStrategy, finalObject,
partCustomTimeStrategy,
finalObject,
new FakeStorageInternal() {
@Override
public BlobInfo internalDirectUpload(
Expand Down Expand Up @@ -432,7 +435,8 @@ public void creatingAnEmptyObjectWhichFailsIsSetAsResultFailureAndThrowFromClose
partNamingStrategy,
PartCleanupStrategy.always(),
3,
partCustomTimeStrategy, finalObject,
partCustomTimeStrategy,
finalObject,
new FakeStorageInternal() {
@Override
public BlobInfo internalDirectUpload(
Expand Down Expand Up @@ -465,7 +469,8 @@ public void badServerCrc32cResultsInException() throws Exception {
partNamingStrategy,
PartCleanupStrategy.always(),
3,
partCustomTimeStrategy, finalObject,
partCustomTimeStrategy,
finalObject,
new FakeStorageInternal() {
@Override
public BlobInfo compose(ComposeRequest composeRequest) {
Expand Down Expand Up @@ -571,7 +576,8 @@ public BlobInfo internalDirectUpload(
partNamingStrategy,
PartCleanupStrategy.never(),
32,
partCustomTimeStrategy, finalObject,
partCustomTimeStrategy,
finalObject,
storageInternal,
info,
opts);
Expand Down Expand Up @@ -650,7 +656,8 @@ public void errorContextIsPopulated() throws Exception {
partNamingStrategy,
PartCleanupStrategy.never(),
3,
partCustomTimeStrategy, finalObject,
partCustomTimeStrategy,
finalObject,
new FakeStorageInternal() {
@Override
public BlobInfo compose(ComposeRequest composeRequest) {
Expand Down Expand Up @@ -732,7 +739,8 @@ public BlobInfo internalObjectGet(BlobId blobId, Opts<ObjectSourceOpt> opts) {
partNamingStrategy,
PartCleanupStrategy.always(),
10,
partCustomTimeStrategy, finalObject,
partCustomTimeStrategy,
finalObject,
storageInternal,
info,
opts);
Expand Down Expand Up @@ -760,7 +768,8 @@ private ParallelCompositeUploadWritableByteChannel defaultPcu(int maxElementsPer
partNamingStrategy,
PartCleanupStrategy.always(),
maxElementsPerCompact,
partCustomTimeStrategy, finalObject,
partCustomTimeStrategy,
finalObject,
storageInternal,
info,
opts);
Expand Down