Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

docs: samples and tests for backup Admin APIs and overall spanner Admin APIs. #2882

Merged
merged 44 commits into from
Feb 23, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
edc5bbf
fix: prevent illegal negative timeout values into thread sleep() meth…
arpan14 Feb 6, 2023
49a85df
Merge pull request #1 from arpan14/retryerror
arpan14 Feb 8, 2023
4cd497b
Fixing lint issues.
arpan14 Feb 8, 2023
4a6aa8e
Merge branch 'googleapis:main' into main
arpan14 Mar 13, 2023
b2aa09d
Merge branch 'googleapis:main' into main
arpan14 Mar 15, 2023
8d6d71e
Merge branch 'googleapis:main' into main
arpan14 May 9, 2023
77e6e7d
Merge branch 'googleapis:main' into main
arpan14 Jul 17, 2023
e8b7fad
Merge branch 'googleapis:main' into main
arpan14 Jul 25, 2023
8aa84e1
Merge branch 'googleapis:main' into main
arpan14 Oct 10, 2023
57fd405
Merge branch 'googleapis:main' into main
arpan14 Oct 27, 2023
1253563
Merge branch 'googleapis:main' into main
arpan14 Nov 20, 2023
d4f6a60
Merge branch 'googleapis:main' into main
arpan14 Dec 15, 2023
3efaf7c
Merge branch 'googleapis:main' into main
arpan14 Dec 26, 2023
f41b39f
Merge branch 'googleapis:main' into main
arpan14 Jan 3, 2024
3775996
chore: copy samples and rewrite create backup.
arpan14 Jan 10, 2024
c9d60df
chore: fix code.
arpan14 Jan 10, 2024
8989e7c
chore: fix code.
arpan14 Jan 10, 2024
1fedc6e
Merge branch 'googleapis:main' into admin-backup-samples
arpan14 Feb 13, 2024
4e84671
chore: fix code.
arpan14 Jan 11, 2024
3c0d0f6
chore: fix error.
arpan14 Feb 13, 2024
8e51bde
fix: all compilation errors.
arpan14 Feb 15, 2024
fb9ad16
fix: more compilation issues.
arpan14 Feb 15, 2024
0182569
fix: all compile issues.
arpan14 Feb 15, 2024
aa8faca
chore: compile and test PgSpannerSample.
arpan14 Feb 16, 2024
b806ef1
fix: all issues with PgSpannerSample.
arpan14 Feb 16, 2024
7d5943d
chore: compile and fix SpannerSample/SpannerSampleIT.
arpan14 Feb 17, 2024
db5cbca
chore: fix kms configs and ITs.
arpan14 Feb 19, 2024
9240fbc
chore: fix cancel backup tests.
arpan14 Feb 19, 2024
e752d32
Update samples/snippets/src/main/java/com/example/spanner/admin/gener…
arpan14 Feb 19, 2024
ca59fb6
chore: add log statements in tests.
arpan14 Feb 20, 2024
fcf4457
chore: fix lint errors and fix comments.
arpan14 Feb 20, 2024
89834e0
chore: fix lint errors.
arpan14 Feb 20, 2024
478c343
chore: fix lint errors.
arpan14 Feb 20, 2024
d719f7f
chore: fix backup samples for restore use-case.
arpan14 Feb 22, 2024
d324ed6
fix: fix restore/list backup tests.
arpan14 Feb 22, 2024
837034a
chore: fix PgSpannerSample sample and test.
arpan14 Feb 22, 2024
f7623ed
chore: fix delete backup test.
arpan14 Feb 23, 2024
3f90c1a
chore: fix lint errors.
arpan14 Feb 23, 2024
bb7a6cc
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Feb 23, 2024
6f36d93
Merge branch 'main' into admin-backup-samples
arpan14 Feb 23, 2024
01058bc
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Feb 23, 2024
0dcce23
chore: add IT for CopyBackupSample.
arpan14 Feb 23, 2024
fa88a45
chore: fix lint errors.
arpan14 Feb 23, 2024
e3630bc
chore: refactor test for delete backup.
arpan14 Feb 23, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Prev Previous commit
Next Next commit
chore: fix code.
  • Loading branch information
arpan14 committed Feb 15, 2024
commit 4e8467111111d154fec8156441d4c5cb21c11fef
Expand Up @@ -29,8 +29,6 @@
import com.google.cloud.spanner.DatabaseClient;
import com.google.cloud.spanner.DatabaseId;
import com.google.cloud.spanner.ErrorCode;
import com.google.cloud.spanner.Instance;
import com.google.cloud.spanner.InstanceId;
import com.google.cloud.spanner.Key;
import com.google.cloud.spanner.KeyRange;
import com.google.cloud.spanner.KeySet;
Expand All @@ -50,6 +48,8 @@
import com.google.cloud.spanner.Type;
import com.google.cloud.spanner.Value;
import com.google.cloud.spanner.admin.database.v1.DatabaseAdminClient;
import com.google.cloud.spanner.admin.database.v1.DatabaseAdminClient.ListBackupOperationsPagedResponse;
import com.google.cloud.spanner.admin.database.v1.DatabaseAdminClient.ListDatabaseOperationsPagedResponse;
import com.google.cloud.spanner.admin.instance.v1.InstanceAdminClient;
import com.google.common.io.BaseEncoding;
import com.google.longrunning.Operation;
Expand All @@ -62,9 +62,13 @@
import com.google.spanner.admin.database.v1.CreateDatabaseRequest;
import com.google.spanner.admin.database.v1.DatabaseName;
import com.google.spanner.admin.database.v1.InstanceName;
import com.google.spanner.admin.database.v1.ListBackupOperationsRequest;
import com.google.spanner.admin.database.v1.ListBackupsRequest;
import com.google.spanner.admin.database.v1.ListDatabaseOperationsRequest;
import com.google.spanner.admin.database.v1.ListDatabaseOperationsResponse;
import com.google.spanner.admin.database.v1.OptimizeRestoredDatabaseMetadata;
import com.google.spanner.admin.database.v1.RestoreDatabaseMetadata;
import com.google.spanner.admin.database.v1.UpdateDatabaseDdlMetadata;
import com.google.spanner.admin.instance.v1.Instance;
import com.google.spanner.v1.ExecuteSqlRequest.QueryOptions;
import java.math.BigDecimal;
import java.time.Instant;
Expand All @@ -73,12 +77,10 @@
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.threeten.bp.Duration;
import org.threeten.bp.LocalDate;
import org.threeten.bp.LocalDateTime;
import org.threeten.bp.OffsetDateTime;
import org.threeten.bp.temporal.ChronoField;
import org.threeten.bp.temporal.TemporalUnit;

/**
* Example code for using the Cloud Spanner API. This example demonstrates all the common operations
Expand Down Expand Up @@ -1242,30 +1244,25 @@ static void updateUsingBatchDml(DatabaseClient dbClient) {
// [END spanner_dml_batch_update]

// [START spanner_create_table_with_datatypes]
static void createTableWithDatatypes(DatabaseAdminClient dbAdminClient, DatabaseId id) {
OperationFuture<Void, UpdateDatabaseDdlMetadata> op =
dbAdminClient.updateDatabaseDdl(
id.getInstanceId().getInstance(),
id.getDatabase(),
Arrays.asList(
"CREATE TABLE Venues ("
+ " VenueId INT64 NOT NULL,"
+ " VenueName STRING(100),"
+ " VenueInfo BYTES(MAX),"
+ " Capacity INT64,"
+ " AvailableDates ARRAY<DATE>,"
+ " LastContactDate DATE,"
+ " OutdoorVenue BOOL, "
+ " PopularityScore FLOAT64, "
+ " Revenue NUMERIC, "
+ " VenueDetails JSON, "
+ " LastUpdateTime TIMESTAMP NOT NULL OPTIONS (allow_commit_timestamp=true)"
+ ") PRIMARY KEY (VenueId)"),
null);
static void createTableWithDatatypes(DatabaseAdminClient dbAdminClient, DatabaseName databaseName) {
try {
// Initiate the request which returns an OperationFuture.
op.get();
System.out.println("Created Venues table in database: [" + id + "]");
dbAdminClient.updateDatabaseDdlAsync(databaseName,
Arrays.asList(
"CREATE TABLE Venues ("
+ " VenueId INT64 NOT NULL,"
+ " VenueName STRING(100),"
+ " VenueInfo BYTES(MAX),"
+ " Capacity INT64,"
+ " AvailableDates ARRAY<DATE>,"
+ " LastContactDate DATE,"
+ " OutdoorVenue BOOL, "
+ " PopularityScore FLOAT64, "
+ " Revenue NUMERIC, "
+ " VenueDetails JSON, "
+ " LastUpdateTime TIMESTAMP NOT NULL OPTIONS (allow_commit_timestamp=true)"
+ ") PRIMARY KEY (VenueId)")).get();
System.out.println("Created Venues table in database: [" + databaseName.toString() + "]");
} catch (ExecutionException e) {
// If the operation failed during execution, expose the cause.
throw (SpannerException) e.getCause();
Expand Down Expand Up @@ -1602,28 +1599,33 @@ static void createBackup(DatabaseAdminClient dbAdminClient, String projectId, St

// [START spanner_cancel_backup_create]
static void cancelCreateBackup(
DatabaseAdminClient dbAdminClient, DatabaseId databaseId, BackupId backupId) {
DatabaseAdminClient dbAdminClient, String projectId, String instanceId,
String databaseId, String backupId) {
// Set expire time to 14 days from now.
Timestamp expireTime = Timestamp.ofTimeMicroseconds(TimeUnit.MICROSECONDS.convert(
System.currentTimeMillis() + TimeUnit.DAYS.toMillis(14), TimeUnit.MILLISECONDS));

// Create a backup instance.
Backup backup =
dbAdminClient
.newBackupBuilder(backupId)
.setDatabase(databaseId)
.setExpireTime(expireTime)
.build();
// Start the creation of a backup.
System.out.println("Creating backup [" + backup.getId() + "]...");
OperationFuture<Backup, CreateBackupMetadata> op = backup.create();
Timestamp expireTime =
Timestamp.newBuilder().setSeconds(TimeUnit.MILLISECONDS.toSeconds((
System.currentTimeMillis() + TimeUnit.DAYS.toMillis(14)))).build();
BackupName backupName = BackupName.of(projectId, instanceId, backupId);
Backup backup = Backup.newBuilder()
.setName(backupName.toString())
.setDatabase(DatabaseName.of(projectId, instanceId, databaseId).toString())
.setExpireTime(expireTime).build();


try {
// Try to cancel the backup operation.
System.out.println("Cancelling create backup operation for [" + backup.getId() + "]...");
dbAdminClient.cancelOperation(op.getName());
// Start the creation of a backup.
System.out.println("Creating backup [" + backupId + "]...");
OperationFuture<Backup, CreateBackupMetadata> op = dbAdminClient.createBackupAsync(
InstanceName.of(projectId, instanceId), backup, backupId);

// Get a polling future for the running operation. This future will regularly poll the server
// for the current status of the backup operation.
RetryingFuture<OperationSnapshot> pollingFuture = op.getPollingFuture();

// Try to cancel the backup operation.
System.out.println("Cancelling create backup operation for [" + backupId + "]...");
pollingFuture.cancel(true);

// Wait for the operation to finish.
// isDone will return true when the operation is complete, regardless of whether it was
// successful or not.
Expand All @@ -1633,11 +1635,11 @@ static void cancelCreateBackup(
}
if (pollingFuture.get().getErrorCode() == null) {
// Backup was created before it could be cancelled. Delete the backup.
backup.delete();
System.out.println("Backup operation for [" + backup.getId()
dbAdminClient.deleteBackup(backupName);
System.out.println("Backup operation for [" + backupId
+ "] successfully finished before it could be cancelled");
} else if (pollingFuture.get().getErrorCode().getCode() == StatusCode.Code.CANCELLED) {
System.out.println("Backup operation for [" + backup.getId() + "] successfully cancelled");
System.out.println("Backup operation for [" + backupId + "] successfully cancelled");
}
} catch (ExecutionException e) {
throw SpannerExceptionFactory.newSpannerException(e.getCause());
Expand All @@ -1649,20 +1651,22 @@ static void cancelCreateBackup(

// [START spanner_list_backup_operations]
static void listBackupOperations(
InstanceAdminClient instanceAdminClient, DatabaseId databaseId, BackupId backupId) {
Instance instance = instanceAdminClient.getInstance(databaseId.getInstanceId().getInstance());
DatabaseAdminClient databaseAdminClient,
String projectId, String instanceId,
String databaseId, String backupId) {
com.google.spanner.admin.instance.v1.InstanceName instanceName = com.google.spanner.admin.instance.v1.InstanceName.of(projectId, instanceId)
// Get create backup operations for the sample database.
arpan14 marked this conversation as resolved.
Show resolved Hide resolved
Timestamp last24Hours = Timestamp.ofTimeSecondsAndNanos(TimeUnit.SECONDS.convert(
TimeUnit.HOURS.convert(Timestamp.now().getSeconds(), TimeUnit.SECONDS) - 24,
TimeUnit.HOURS), 0);
String filter =
String.format(
"(metadata.@type:type.googleapis.com/"
+ "google.spanner.admin.database.v1.CreateBackupMetadata) "
+ "AND (metadata.database:%s)",
databaseId.getName());
Page<Operation> createBackupOperations = instance.listBackupOperations(
Options.filter(filter));
DatabaseName.of(projectId, instanceId, databaseId).toString());
ListBackupOperationsRequest listBackupOperationsRequest =
ListBackupOperationsRequest.newBuilder()
.setParent(instanceName.toString()).setFilter(filter).build();
ListBackupOperationsPagedResponse createBackupOperations
= databaseAdminClient.listBackupOperations(listBackupOperationsRequest);
System.out.println("Create Backup Operations:");
for (Operation op : createBackupOperations.iterateAll()) {
try {
Expand All @@ -1679,13 +1683,15 @@ static void listBackupOperations(
}
}
// Get copy backup operations for the sample database.
filter =
String.format(
filter = String.format(
"(metadata.@type:type.googleapis.com/"
+ "google.spanner.admin.database.v1.CopyBackupMetadata) "
+ "AND (metadata.source_backup:%s)",
backupId.getName());
Page<Operation> copyBackupOperations = instance.listBackupOperations(Options.filter(filter));
BackupName.of(projectId, instanceId, backupId).toString());
listBackupOperationsRequest =
ListBackupOperationsRequest.newBuilder()
.setParent(instanceName.toString()).setFilter(filter).build();
ListBackupOperationsPagedResponse copyBackupOperations = databaseAdminClient.listBackupOperations(listBackupOperationsRequest);
System.out.println("Copy Backup Operations:");
for (Operation op : copyBackupOperations.iterateAll()) {
try {
Expand All @@ -1707,18 +1713,22 @@ static void listBackupOperations(

// [START spanner_list_database_operations]
static void listDatabaseOperations(
InstanceAdminClient instanceAdminClient,
DatabaseAdminClient dbAdminClient,
InstanceId instanceId) {
Instance instance = instanceAdminClient.getInstance(instanceId.getInstance());
DatabaseAdminClient dbAdminClient, String projectId, String instanceId) {
// Get optimize restored database operations.
Timestamp last24Hours = Timestamp.ofTimeSecondsAndNanos(TimeUnit.SECONDS.convert(
TimeUnit.HOURS.convert(Timestamp.now().getSeconds(), TimeUnit.SECONDS) - 24,
TimeUnit.HOURS), 0);
Timestamp last24Hours = Timestamp.newBuilder().setSeconds((TimeUnit.SECONDS.convert(
TimeUnit.HOURS.convert(new java.sql.Timestamp(
System.currentTimeMillis()).getSeconds(), TimeUnit.SECONDS) - 24,
TimeUnit.HOURS))).build();
arpan14 marked this conversation as resolved.
Show resolved Hide resolved
String filter = String.format("(metadata.@type:type.googleapis.com/"
+ "google.spanner.admin.database.v1.OptimizeRestoredDatabaseMetadata) AND "
+ "(metadata.progress.start_time > \"%s\")", last24Hours);
for (Operation op : instance.listDatabaseOperations(Options.filter(filter)).iterateAll()) {
ListDatabaseOperationsRequest listDatabaseOperationsRequest =
ListDatabaseOperationsRequest.newBuilder()
.setParent(com.google.spanner.admin.instance.v1.InstanceName.of(
projectId, instanceId).toString()).setFilter(filter).build();
ListDatabaseOperationsPagedResponse pagedResponse
= dbAdminClient.listDatabaseOperations(listDatabaseOperationsRequest);
for (Operation op : pagedResponse.iterateAll()) {
try {
OptimizeRestoredDatabaseMetadata metadata =
op.getMetadata().unpack(OptimizeRestoredDatabaseMetadata.class);
Expand All @@ -1736,46 +1746,60 @@ static void listDatabaseOperations(

// [START spanner_list_backups]
static void listBackups(
InstanceAdminClient instanceAdminClient, DatabaseId databaseId, BackupId backupId) {
Instance instance = instanceAdminClient.getInstance(databaseId.getInstanceId().getInstance());
DatabaseAdminClient dbAdminClient, String projectId,
String instanceId, String databaseId, String backupId) {
InstanceName instanceName = InstanceName.of(projectId, instanceId)
// List all backups.
System.out.println("All backups:");
for (Backup backup : instance.listBackups().iterateAll()) {
for (Backup backup : dbAdminClient.listBackups(
instanceName.toString()).iterateAll()) {
System.out.println(backup);
}

// List all backups with a specific name.
System.out.println(
String.format("All backups with backup name containing \"%s\":", backupId.getBackup()));
for (Backup backup : instance.listBackups(
Options.filter(String.format("name:%s", backupId.getBackup()))).iterateAll()) {
String.format("All backups with backup name containing \"%s\":", backupId));
ListBackupsRequest listBackupsRequest =
ListBackupsRequest.newBuilder().setParent(instanceName.toString()).setFilter(String.format("name:%s", backupId)).build();
for (Backup backup : dbAdminClient.listBackups(listBackupsRequest).iterateAll()) {
System.out.println(backup);
}

// List all backups for databases whose name contains a certain text.
System.out.println(
String.format(
"All backups for databases with a name containing \"%s\":",
databaseId.getDatabase()));
for (Backup backup : instance.listBackups(
Options.filter(String.format("database:%s", databaseId.getDatabase()))).iterateAll()) {
"All backups for databases with a name containing \"%s\":", databaseId));
listBackupsRequest =
ListBackupsRequest.newBuilder().setParent(instanceName.toString())
.setFilter(String.format("database:%s", databaseId)).build();
for (Backup backup : dbAdminClient.listBackups(listBackupsRequest).iterateAll()) {
System.out.println(backup);
}

// List all backups that expire before a certain time.
Timestamp expireTime = Timestamp.ofTimeMicroseconds(TimeUnit.MICROSECONDS.convert(
Timestamp expireTime = Timestamp.newBuilder().setSeconds((TimeUnit.SECONDS.convert(
TimeUnit.SECONDS.convert(new java.sql.Timestamp(
System.currentTimeMillis()).getSeconds(), TimeUnit.SECONDS) + TimeUnit.DAYS.toSeconds(30),
TimeUnit.HOURS))).build();

Timestamp expireTime = Timestamp.ofTimeMicroseconds(TimeUnit.SECONDS.convert(
System.currentTimeMillis() + TimeUnit.DAYS.toMillis(30), TimeUnit.MILLISECONDS));
System.out.println(String.format("All backups that expire before %s:", expireTime.toString()));
for (Backup backup :
instance.listBackups(
Options.filter(String.format("expire_time < \"%s\"", expireTime.toString())))
.iterateAll()) {
listBackupsRequest =
ListBackupsRequest.newBuilder().setParent(instanceName.toString())
.setFilter(String.format("expire_time < \"%s\"", expireTime.toString())).build();

for (Backup backup : dbAdminClient.listBackups(listBackupsRequest).iterateAll()) {
System.out.println(backup);
}

// List all backups with size greater than a certain number of bytes.
listBackupsRequest =
ListBackupsRequest.newBuilder().setParent(instanceName.toString())
.setFilter("size_bytes > 100").build();

System.out.println("All backups with size greater than 100 bytes:");
for (Backup backup : instance.listBackups(Options.filter("size_bytes > 100")).iterateAll()) {
for (Backup backup : dbAdminClient.listBackups(listBackupsRequest).iterateAll()) {
System.out.println(backup);
}

Expand Down Expand Up @@ -2033,7 +2057,8 @@ static void run(
updateUsingBatchDml(dbClient);
break;
case "createtablewithdatatypes":
createTableWithDatatypes(dbAdminClient, database);
createTableWithDatatypes(dbAdminClient, DatabaseName.of(database.getInstanceId().getProject(),
database.getInstanceId().getInstance(), database.getDatabase()));
break;
case "writedatatypesdata":
writeDatatypesData(dbClient);
Expand Down Expand Up @@ -2079,7 +2104,9 @@ static void run(
case "cancelcreatebackup":
cancelCreateBackup(
dbAdminClient,
database,
database.getInstanceId().getProject(),
database.getInstanceId().getInstance(), database.getDatabase(),
backupId
BackupId.of(backup.getInstanceId(), backup.getBackup() + "_cancel"));
break;
case "listbackupoperations":
Expand All @@ -2099,10 +2126,12 @@ static void run(
DatabaseId.of(database.getInstanceId(), createRestoredSampleDbId(database)));
break;
case "updatebackup":
updateBackup(dbAdminClient, backup);
updateBackup(dbAdminClient, database.getInstanceId().getProject(),
database.getInstanceId().getInstance(), backupId);
break;
case "deletebackup":
deleteBackup(dbAdminClient, backup);
deleteBackup(dbAdminClient, database.getInstanceId().getProject(),
database.getInstanceId().getInstance(), backupId);
break;
default:
printUsageAndExit();
Expand Down
Expand Up @@ -14,10 +14,12 @@
* limitations under the License.
*/

package com.example.spanner;
package com.example.spanner.admin.generated;

import static com.google.common.truth.Truth.assertThat;

import com.example.spanner.SampleRunner;
import com.example.spanner.SampleTestBase;
import com.google.cloud.spanner.ErrorCode;
import com.google.cloud.spanner.SpannerException;
import com.google.cloud.spanner.SpannerExceptionFactory;
Expand All @@ -32,8 +34,8 @@
import org.junit.runners.JUnit4;

/**
* Integration tests for: {@link CreateDatabaseWithEncryptionKey}, {@link
* CreateBackupWithEncryptionKey} and {@link RestoreBackupWithEncryptionKey}
* Integration tests for: {@link com.example.spanner.CreateDatabaseWithEncryptionKey}, {@link
* com.example.spanner.CreateBackupWithEncryptionKey} and {@link com.example.spanner.RestoreBackupWithEncryptionKey}
*/
@RunWith(JUnit4.class)
@Ignore
Expand Down
Expand Up @@ -14,7 +14,7 @@
* limitations under the License.
*/

package com.example.spanner;
package com.example.spanner.admin.generated;

import static com.google.common.truth.Truth.assertThat;

Expand Down