Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: implement query profile #1365

Merged
merged 19 commits into from
Mar 25, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Prev Previous commit
Next Next commit
cleanup, add docs
  • Loading branch information
kolea2 committed Mar 13, 2024
commit a439264d9a491901bfd60a1c95bb6cc81bbd826a
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@ public int size() {
return this.aggregationResults.size();
}

/*
* Returns the ResultSetStats if QueryMode is set to EXPLAIN or EXPLAIN_ANALYZE. Otherwise, returns null.
/**
* Returns {@code ExplainMetrics} if {@code ExplainOptions} were enabled. Otherwise, returns null.
*/
@BetaApi
public Optional<ExplainMetrics> getExplainMetrics() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -475,8 +475,7 @@ interface TransactionCallable<T> {
* StructuredQuery<Entity> query = Query.newEntityQueryBuilder()
* .setKind(kind)
* .build();
* QueryResults<Entity> results = datastore.run(query, QueryMode.EXPLAIN_ANALYZE);
* ResultSetStats resultSetStats = results.getResultSetStats();
* QueryResults<Entity> results = datastore.run(query, ExplainOptions.newBuilder().setAnalyze(true).build());
* }</pre>
*
* @throws DatastoreException upon failure
Expand Down Expand Up @@ -551,8 +550,7 @@ default AggregationResults runAggregation(AggregationQuery query, ReadOption...
* .addAggregation(count().as("total_count"))
* .over(selectAllQuery)
* .build();
* AggregationResults aggregationResults = datastore.runAggregation(aggregationQuery, QueryMode.EXPLAIN_ANALYZE);
* ResultSetStats aggregationStats = aggregationResults.getResultSetStats();
* AggregationResults aggregationResults = datastore.runAggregation(aggregationQuery, ExplainOptions.newBuilder().setAnalyze(true).build());
* }</pre>
*
* @throws DatastoreException upon failure
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,6 @@ public interface QueryResults<V> extends Iterator<V> {

@BetaApi
default Optional<ExplainMetrics> getExplainMetrics() {
throw new UnsupportedOperationException("not implemented");
throw new UnsupportedOperationException("Not implemented.");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ interface Response {

@BetaApi
default <T> QueryResults<T> run(Query<T> query, ExplainOptions explainOptions) {
throw new UnsupportedOperationException("not implemented");
throw new UnsupportedOperationException("Not implemented.");
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,19 @@
*/
package com.google.cloud.datastore.models;

import com.google.api.core.BetaApi;
import com.google.api.core.InternalApi;
import com.google.cloud.Structs;
import com.google.common.base.Objects;
import java.util.Map;
import org.threeten.bp.Duration;

/** Model class for {@code com.google.datastore.v1.ExecutionStats} */
@BetaApi
public class ExecutionStats {

private final long resultsReturned;

private final Duration executionDuration;

private final long readOperations;

private final Map<String, Object> debugStats;

@InternalApi
Expand All @@ -39,18 +38,25 @@ public ExecutionStats(com.google.datastore.v1.ExecutionStats proto) {
this.debugStats = Structs.asMap(proto.getDebugStats());
}

/**
* Returns the total number of results returned, including documents, projections, aggregation
* results, keys.
*/
public long getResultsReturned() {
return resultsReturned;
}

/** Returns the debugging statistics from the execution of the query. */
public Map<String, Object> getDebugStats() {
return debugStats;
}

/** Returns the total time to execute the query in the backend. */
public Duration getExecutionDuration() {
return executionDuration;
}

/** Returns the total billable read operations. */
public long getReadOperations() {
return readOperations;
}
Expand All @@ -68,7 +74,7 @@ public boolean equals(Object o) {
return Objects.equal(resultsReturned, that.resultsReturned)
&& Objects.equal(executionDuration, that.executionDuration)
&& Objects.equal(readOperations, that.readOperations)
&& java.util.Objects.equals(debugStats, that.debugStats);
&& Objects.equal(debugStats, that.debugStats);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,15 @@
*/
package com.google.cloud.datastore.models;

import com.google.api.core.BetaApi;
import com.google.api.core.InternalApi;
import com.google.common.base.Objects;
import java.util.Optional;

/** Model class for {@code com.google.datastore.v1.ExplainMetrics}. */
@BetaApi
public class ExplainMetrics {
private PlanSummary planSummary;
private final PlanSummary planSummary;
private ExecutionStats executionStats;

@InternalApi
Expand All @@ -31,22 +34,19 @@ public ExplainMetrics(com.google.datastore.v1.ExplainMetrics proto) {
this.planSummary = new PlanSummary(proto.getPlanSummary());
}

/** Returns the planning phase information for the query. */
public PlanSummary getPlanSummary() {
return planSummary;
}

public void setPlanSummary(PlanSummary planSummary) {
this.planSummary = planSummary;
}

/**
* Returns the aggregated stats from the execution of the query, if present. Only present when
* 'analyze' is set to true for {@code ExplainOptions}.
*/
public Optional<ExecutionStats> getExecutionStats() {
return Optional.ofNullable(executionStats);
}

public void setExecutionStats(ExecutionStats executionStats) {
this.executionStats = executionStats;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,16 @@
*/
package com.google.cloud.datastore.models;

import com.google.api.core.BetaApi;
import com.google.api.core.InternalApi;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Objects;

/**
* Model class for {@code com.google.datastore.v1.ExplainOptions}. Contains the explain options for
* the query. Analyze is set to 'false' by default.
*/
@BetaApi
public class ExplainOptions {
private final com.google.datastore.v1.ExplainOptions proto;

Expand All @@ -30,6 +36,12 @@ public static Builder newBuilder() {
return new Builder();
}

/**
* Returns whether analyze is set to true or false. When false (the default), the query will be
* planned, returning only metrics from the planning stages. When true, the query will be planned
* and executed, returning the full query results along with both planning and execution stage
* metrics.
*/
public boolean shouldAnalyze() {
return proto.getAnalyze();
}
Expand All @@ -56,6 +68,11 @@ public com.google.datastore.v1.ExplainOptions toPb() {
public static class Builder {
private boolean analyze = false;

/*
* Set 'analyze' to true or false for the explain options.
* When false (the default), the query will be planned, returning only metrics from the planning stages.
* When true, the query will be planned and executed, returning the full query results along with both planning and execution stage metrics.
*/
public Builder setAnalyze(boolean analyze) {
this.analyze = analyze;
return this;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,16 @@
*/
package com.google.cloud.datastore.models;

import com.google.api.core.BetaApi;
import com.google.api.core.InternalApi;
import com.google.cloud.Structs;
import com.google.common.base.Objects;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

/** Model class for {@code com.google.datastore.v1.PlanSummary} */
@BetaApi
public class PlanSummary {
private final List<Map<String, Object>> indexesUsed = new ArrayList<>();

Expand All @@ -32,6 +35,7 @@ public PlanSummary(com.google.datastore.v1.PlanSummary proto) {
.forEach(indexesUsed -> this.indexesUsed.add(Structs.asMap(indexesUsed)));
}

/** Returns the indexes selected for the query. */
public List<Map<String, Object>> getIndexesUsed() {
return indexesUsed;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,12 @@
import com.google.cloud.datastore.models.ExplainMetrics;
import com.google.common.collect.ImmutableMap;
import com.google.datastore.v1.AggregationResultBatch;
import com.google.datastore.v1.ExecutionStats;
import com.google.datastore.v1.PlanSummary;
import com.google.datastore.v1.RunAggregationQueryResponse;
import com.google.datastore.v1.Value;
import com.google.protobuf.Duration;
import com.google.protobuf.Struct;
import java.util.AbstractMap.SimpleEntry;
import java.util.HashMap;
import java.util.Map;
Expand Down Expand Up @@ -108,8 +112,34 @@ public void shouldTransformAggregationQueryResponseWithIntValuesWithStats() {
.setReadTime(readTime.toProto())
.build();

ExecutionStats executionStats =
ExecutionStats.newBuilder()
.setDebugStats(
Struct.newBuilder()
.putFields(
"field",
com.google.protobuf.Value.newBuilder().setStringValue("val").build())
.build())
.setExecutionDuration(Duration.newBuilder().setSeconds(1).build())
.setReadOperations(1)
.setResultsReturned(2)
.build();

PlanSummary planSummary =
PlanSummary.newBuilder()
.addIndexesUsed(
Struct.newBuilder()
.putFields(
"field2",
com.google.protobuf.Value.newBuilder().setStringValue("val2").build())
.build())
.build();

com.google.datastore.v1.ExplainMetrics explainMetrics =
com.google.datastore.v1.ExplainMetrics.newBuilder().build();
com.google.datastore.v1.ExplainMetrics.newBuilder()
.setExecutionStats(executionStats)
.setPlanSummary(planSummary)
.build();

RunAggregationQueryResponse runAggregationQueryResponse =
RunAggregationQueryResponse.newBuilder()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,12 @@ public void testQueryProfile() {
Truth.assertThat(results3.hasNext()).isTrue();
Truth.assertThat(results3.getExplainMetrics().isPresent()).isFalse();

QueryResults<Entity> results4 =
datastore.run(simpleOrQuery, ExplainOptions.newBuilder().setAnalyze(false).build());
Truth.assertThat(results4.hasNext()).isFalse();
assertPlanSummary(results4.getExplainMetrics().get().getPlanSummary());
Truth.assertThat(results4.getExplainMetrics().get().getExecutionStats().isPresent()).isFalse();

AggregationQuery aggregationQuery =
Query.newAggregationQueryBuilder().over(simpleOrQuery).addAggregation(count()).build();
AggregationResults resultsAggregation =
Expand Down Expand Up @@ -529,7 +535,7 @@ public void testTransactionWithQuery() throws Exception {
}

@Test
public void testTransactionQueryModeExplainAnalyze() {
public void testTransactionExplainOptionsAnalyze() {
StructuredQuery<Entity> query =
Query.newEntityQueryBuilder()
.setKind(KIND2)
Expand Down Expand Up @@ -566,7 +572,7 @@ public void testTransactionQueryModeExplainAnalyze() {
}

@Test
public void testTransactionQueryModeExplain() {
public void testTransactionExplainOptions() {
StructuredQuery<Entity> query =
Query.newEntityQueryBuilder()
.setKind(KIND2)
Expand Down Expand Up @@ -598,20 +604,21 @@ public void testTransactionQueryModeExplain() {
private void assertPlanSummary(PlanSummary planSummary) {
List<Map<String, Object>> indexesUsed = planSummary.getIndexesUsed();
indexesUsed.forEach(
each -> {
Truth.assertThat(each.keySet()).containsAtLeast("properties", "query_scope");
});
each -> Truth.assertThat(each.keySet()).containsAtLeast("properties", "query_scope"));
}

private void assertExecutionStats(ExecutionStats executionStats) {
Map<String, Object> queryStatsAggregation = executionStats.getDebugStats();
Map<String, Object> debugStats = executionStats.getDebugStats();
Truth.assertThat(debugStats.keySet())
.containsAtLeast("billing_details", "documents_scanned", "index_entries_scanned");

Duration executionDuration = executionStats.getExecutionDuration();
Truth.assertThat(executionDuration).isIn(Range.greaterThan(Duration.ofMillis(0)));

long readOperations = executionStats.getReadOperations();
long resultsReturned = executionStats.getResultsReturned();
Truth.assertThat(queryStatsAggregation.keySet())
.containsAtLeast("billing_details", "documents_scanned", "index_entries_scanned");
Truth.assertThat(executionDuration).isIn(Range.atLeast(Duration.ofMillis(0)));
Truth.assertThat(readOperations).isIn(Range.atLeast(1L));

long resultsReturned = executionStats.getResultsReturned();
Truth.assertThat(resultsReturned).isIn(Range.atLeast(1L));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ public void testModel() {
public void testEqualsAndHashcode() {
com.google.datastore.v1.ExecutionStats proto2 =
com.google.datastore.v1.ExecutionStats.newBuilder()
.setBytesReturned(5)
.setDebugStats(struct)
.setExecutionDuration(duration)
.setReadOperations(6)
Expand Down