From 7da5a7da86ad649a8132e3183f4b3e3f9bb2eace Mon Sep 17 00:00:00 2001 From: Denis DelGrosso <85250797+ddelgrosso1@users.noreply.github.com> Date: Mon, 18 Mar 2024 13:11:39 -0400 Subject: [PATCH] feat: add ability to configure and utilize soft-delete and restore (#2425) --- src/bucket.ts | 11 ++++++ src/file.ts | 69 +++++++++++++++++++++++++++++++++++++ system-test/storage.ts | 78 ++++++++++++++++++++++++++++++++++++++++++ test/bucket.ts | 19 ++++++++++ test/file.ts | 21 ++++++++++++ 5 files changed, 198 insertions(+) diff --git a/src/bucket.ts b/src/bucket.ts index 6596a59e6..599620834 100644 --- a/src/bucket.ts +++ b/src/bucket.ts @@ -169,6 +169,7 @@ export interface GetFilesOptions { maxApiCalls?: number; maxResults?: number; pageToken?: string; + softDeleted?: boolean; startOffset?: string; userProject?: string; versions?: boolean; @@ -342,6 +343,10 @@ export interface BucketMetadata extends BaseMetadata { retentionPeriod?: string | number; } | null; rpo?: string; + softDeletePolicy?: { + retentionDurationSeconds?: string | number; + readonly effectiveTime?: string; + }; storageClass?: string; timeCreated?: string; updated?: string; @@ -2629,6 +2634,9 @@ class Bucket extends ServiceObject { * or 1 page of results will be returned per call. * @property {string} [pageToken] A previously-returned page token * representing part of the larger set of results to view. + * @property {boolean} [softDeleted] If true, only soft-deleted object versions will be + * listed as distinct results in order of generation number. Note `soft_deleted` and + * `versions` cannot be set to true simultaneously. * @property {string} [startOffset] Filter results to objects whose names are * lexicographically equal to or after startOffset. If endOffset is also set, * the objects listed have names between startOffset (inclusive) and endOffset (exclusive). @@ -2671,6 +2679,9 @@ class Bucket extends ServiceObject { * or 1 page of results will be returned per call. * @param {string} [query.pageToken] A previously-returned page token * representing part of the larger set of results to view. + * @param {boolean} [query.softDeleted] If true, only soft-deleted object versions will be + * listed as distinct results in order of generation number. Note `soft_deleted` and + * `versions` cannot be set to true simultaneously. * @param {string} [query.startOffset] Filter results to objects whose names are * lexicographically equal to or after startOffset. If endOffset is also set, * the objects listed have names between startOffset (inclusive) and endOffset (exclusive). diff --git a/src/file.ts b/src/file.ts index b4a4b3aeb..1df546b93 100644 --- a/src/file.ts +++ b/src/file.ts @@ -72,6 +72,8 @@ import { BaseMetadata, DeleteCallback, DeleteOptions, + GetResponse, + InstanceResponseCallback, RequestResponse, SetMetadataOptions, } from './nodejs-common/service-object.js'; @@ -172,6 +174,8 @@ export interface GetFileMetadataCallback { export interface GetFileOptions extends GetConfig { userProject?: string; + generation?: number; + softDeleted?: boolean; } export type GetFileResponse = [File, unknown]; @@ -418,6 +422,11 @@ export interface SetStorageClassCallback { (err?: Error | null, apiResponse?: unknown): void; } +export interface RestoreOptions extends PreconditionOptions { + generation: number; + projection?: 'full' | 'noAcl'; +} + export interface FileMetadata extends BaseMetadata { acl?: AclMetadata[] | null; bucket?: string; @@ -436,6 +445,7 @@ export interface FileMetadata extends BaseMetadata { eventBasedHold?: boolean | null; readonly eventBasedHoldReleaseTime?: string; generation?: string | number; + hardDeleteTime?: string; kmsKeyName?: string; md5Hash?: string; mediaLink?: string; @@ -454,6 +464,7 @@ export interface FileMetadata extends BaseMetadata { } | null; retentionExpirationTime?: string; size?: string | number; + softDeleteTime?: string; storageClass?: string; temporaryHold?: boolean | null; timeCreated?: string; @@ -803,6 +814,9 @@ class File extends ServiceObject { * @param {options} [options] Configuration options. * @param {string} [options.userProject] The ID of the project which will be * billed for the request. + * @param {number} [options.generation] The generation number to get + * @param {boolean} [options.softDeleted] If true, returns the soft-deleted object. + Object `generation` is required if `softDeleted` is set to True. * @param {GetFileCallback} [callback] Callback function. * @returns {Promise} * @@ -2344,6 +2358,27 @@ class File extends ServiceObject { return this; } + get(options?: GetFileOptions): Promise>; + get(callback: InstanceResponseCallback): void; + get(options: GetFileOptions, callback: InstanceResponseCallback): void; + get( + optionsOrCallback?: GetFileOptions | InstanceResponseCallback, + cb?: InstanceResponseCallback + ): Promise> | void { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const options: any = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + cb = + typeof optionsOrCallback === 'function' + ? (optionsOrCallback as InstanceResponseCallback) + : cb; + + super + .get(options) + .then(resp => cb!(null, ...resp)) + .catch(cb!); + } + getExpirationDate(): Promise; getExpirationDate(callback: GetExpirationDateCallback): void; /** @@ -3597,6 +3632,39 @@ class File extends ServiceObject { this.move(destinationFile, options, callback); } + /** + * @typedef {object} RestoreOptions Options for File#restore(). See an + * {@link https://cloud.google.com/storage/docs/json_api/v1/objects#resource| Object resource}. + * @param {string} [userProject] The ID of the project which will be + * billed for the request. + * @param {number} [generation] If present, selects a specific revision of this object. + * @param {string} [projection] Specifies the set of properties to return. If used, must be 'full' or 'noAcl'. + * @param {string | number} [ifGenerationMatch] Request proceeds if the generation of the target resource + * matches the value used in the precondition. + * If the values don't match, the request fails with a 412 Precondition Failed response. + * @param {string | number} [ifGenerationNotMatch] Request proceeds if the generation of the target resource does + * not match the value used in the precondition. If the values match, the request fails with a 304 Not Modified response. + * @param {string | number} [ifMetagenerationMatch] Request proceeds if the meta-generation of the target resource + * matches the value used in the precondition. + * If the values don't match, the request fails with a 412 Precondition Failed response. + * @param {string | number} [ifMetagenerationNotMatch] Request proceeds if the meta-generation of the target resource does + * not match the value used in the precondition. If the values match, the request fails with a 304 Not Modified response. + */ + /** + * Restores a soft-deleted file + * @param {RestoreOptions} options Restore options. + * @returns {Promise} + */ + async restore(options: RestoreOptions): Promise { + const [file] = await this.request({ + method: 'POST', + uri: '/restore', + qs: options, + }); + + return file as File; + } + request(reqOpts: DecorateRequestOptions): Promise; request( reqOpts: DecorateRequestOptions, @@ -4240,6 +4308,7 @@ promisifyAll(File, { 'setEncryptionKey', 'shouldRetryBasedOnPreconditionAndIdempotencyStrat', 'getBufferFromReadable', + 'restore', ], }); diff --git a/system-test/storage.ts b/system-test/storage.ts index 2e0d3ea70..5bee90e5e 100644 --- a/system-test/storage.ts +++ b/system-test/storage.ts @@ -773,6 +773,10 @@ describe('storage', function () { beforeEach(createBucket); + afterEach(async () => { + await bucket.delete(); + }); + it("sets bucket's RPO to ASYNC_TURBO", async () => { await setTurboReplication(bucket, RPO_ASYNC_TURBO); const [bucketMetadata] = await bucket.getMetadata(); @@ -786,6 +790,80 @@ describe('storage', function () { }); }); + describe('soft-delete', () => { + let bucket: Bucket; + const SOFT_DELETE_RETENTION_SECONDS = 7 * 24 * 60 * 60; //7 days in seconds; + + beforeEach(async () => { + bucket = storage.bucket(generateName()); + await bucket.create(); + await bucket.setMetadata({ + softDeletePolicy: { + retentionDurationSeconds: SOFT_DELETE_RETENTION_SECONDS, + }, + }); + }); + + afterEach(async () => { + await bucket.deleteFiles({force: true, versions: true}); + await bucket.delete(); + }); + + it('should set softDeletePolicy correctly', async () => { + const metadata = await bucket.getMetadata(); + assert(metadata[0].softDeletePolicy); + assert(metadata[0].softDeletePolicy.effectiveTime); + assert.deepStrictEqual( + metadata[0].softDeletePolicy.retentionDurationSeconds, + SOFT_DELETE_RETENTION_SECONDS.toString() + ); + }); + + it('should LIST soft-deleted files', async () => { + const f1 = bucket.file('file1'); + const f2 = bucket.file('file2'); + await f1.save('file1'); + await f2.save('file2'); + await f1.delete(); + await f2.delete(); + const [notSoftDeletedFiles] = await bucket.getFiles(); + assert.strictEqual(notSoftDeletedFiles.length, 0); + const [softDeletedFiles] = await bucket.getFiles({softDeleted: true}); + assert.strictEqual(softDeletedFiles.length, 2); + }); + + it('should GET a soft-deleted file', async () => { + const f1 = bucket.file('file3'); + await f1.save('file3'); + const [metadata] = await f1.getMetadata(); + await f1.delete(); + const [softDeletedFile] = await f1.get({ + softDeleted: true, + generation: parseInt(metadata.generation?.toString() || '0'), + }); + assert(softDeletedFile); + assert.strictEqual( + softDeletedFile.metadata.generation, + metadata.generation + ); + }); + + it('should restore a soft-deleted file', async () => { + const f1 = bucket.file('file4'); + await f1.save('file4'); + const [metadata] = await f1.getMetadata(); + await f1.delete(); + let [files] = await bucket.getFiles(); + assert.strictEqual(files.length, 0); + const restoredFile = await f1.restore({ + generation: parseInt(metadata.generation?.toString() || '0'), + }); + assert(restoredFile); + [files] = await bucket.getFiles(); + assert.strictEqual(files.length, 1); + }); + }); + describe('dual-region', () => { let bucket: Bucket; diff --git a/test/bucket.ts b/test/bucket.ts index e2c45af28..3b81ce170 100644 --- a/test/bucket.ts +++ b/test/bucket.ts @@ -1863,6 +1863,25 @@ describe('Bucket', () => { }); }); + it('should return soft-deleted Files if queried for softDeleted', done => { + const softDeletedTime = new Date('1/1/2024').toISOString(); + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, { + items: [{name: 'fake-file-name', generation: 1, softDeletedTime}], + }); + }; + + bucket.getFiles({softDeleted: true}, (err: Error, files: FakeFile[]) => { + assert.ifError(err); + assert(files[0] instanceof FakeFile); + assert.strictEqual(files[0].metadata.softDeletedTime, softDeletedTime); + done(); + }); + }); + it('should set kmsKeyName on file', done => { const kmsKeyName = 'kms-key-name'; diff --git a/test/file.ts b/test/file.ts index be1e2ef74..73de841dd 100644 --- a/test/file.ts +++ b/test/file.ts @@ -104,6 +104,7 @@ const fakePromisify = { 'setEncryptionKey', 'shouldRetryBasedOnPreconditionAndIdempotencyStrat', 'getBufferFromReadable', + 'restore', ]); }, }; @@ -4145,6 +4146,26 @@ describe('File', () => { }); }); + describe('restore', () => { + it('should pass options to underlying request call', async () => { + file.parent.request = function ( + reqOpts: DecorateRequestOptions, + callback_: Function + ) { + assert.strictEqual(this, file); + assert.deepStrictEqual(reqOpts, { + method: 'POST', + uri: '/restore', + qs: {generation: 123}, + }); + assert.strictEqual(callback_, undefined); + return []; + }; + + await file.restore({generation: 123}); + }); + }); + describe('request', () => { it('should call the parent request function', () => { const options = {};