feat: migrate built in metrics to OTEL (#1796)

* feat: migrate exporter to OTEL * address comments * filter out only bigtable metrics * fix test * use the bom * update * update * feat: migrate builtin metrics to OTEl * update completeResultCode * add a comment * udpate * fix tests * remove unrelated changes * fix tests * add documentation * fix test * merge exporter changes * address comments * rebase on otel * revert changes in stats * fix import * update * merge back the endpoint change * refactor constants and settings * refactor and fix tests * remove unused dependency * add some javadoc * address part of the comments * update test * test with nano * measure everything in nanos and publish with double histogram * address comments * fix test * add toString
googleapis · mutianf · Apr 12, 2024 · Jan 23, 2024 · Feb 5, 2024 · Feb 16, 2024
commit 08e171985c11aec0a9af57d02e1014529bffc7bd
@@ -156,6 +156,12 @@
  <differenceType>8001</differenceType>
  <className>com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerBatchedUnaryCallable</className>
  </difference>
+ <!-- InternalApi constructor was updated -->
+ <difference>
+ <differenceType>7004</differenceType>
+ <className>com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory</className>
+ <method>*</method>
+ </difference>
  <!-- InternalApi was updated -->
  <difference>
  <differenceType>6001</differenceType>

@@ -64,19 +64,6 @@
  <dependencies>
  <!-- NOTE: Dependencies are organized into two groups, production and test.
  Within a group, dependencies are sorted by (groupId, artifactId) -->
- <dependency>
- <groupId>com.google.cloud</groupId>
- <artifactId>google-cloud-bigtable-stats</artifactId>
- <!-- Exclude all dependencies that have been shaded. This is to workaround maven's immutable project structure:
- after shading, the maven-shade-plugin tries to remove shaded dependencies, but it can't since the project
- structure is immutable. So we have to manually exclude the shaded transitive dependencies manually. -->
- <exclusions>
- <exclusion>
- <groupId>io.opencensus</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
  <!-- Production dependencies -->
  <dependency>
  <groupId>com.google.api</groupId>
@@ -339,6 +326,10 @@
  <groupId>io.opentelemetry</groupId>
  <artifactId>opentelemetry-api</artifactId>
  </dependency>
+ <dependency>
+ <groupId>io.opentelemetry</groupId>
+ <artifactId>opentelemetry-sdk</artifactId>
+ </dependency>
  <dependency>
  <groupId>io.opentelemetry</groupId>
  <artifactId>opentelemetry-sdk-metrics</artifactId>
@@ -347,6 +338,11 @@
  <groupId>io.opentelemetry</groupId>
  <artifactId>opentelemetry-sdk-common</artifactId>
  </dependency>
+ <dependency>
+ <groupId>io.opentelemetry</groupId>
+ <artifactId>opentelemetry-sdk-testing</artifactId>
+ <scope>test</scope>
+ </dependency>
  <dependency>
  <groupId>com.google.cloud</groupId>
  <artifactId>google-cloud-monitoring</artifactId>

@@ -25,19 +25,16 @@
 import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
 import com.google.api.gax.rpc.UnaryCallSettings;
 import com.google.auth.Credentials;
-import com.google.auth.oauth2.GoogleCredentials;
 import com.google.cloud.bigtable.data.v2.models.Query;
 import com.google.cloud.bigtable.data.v2.models.Row;
 import com.google.cloud.bigtable.data.v2.stub.BigtableBatchingCallSettings;
 import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings;
-import com.google.cloud.bigtable.stats.BigtableStackdriverStatsExporter;
-import com.google.cloud.bigtable.stats.BuiltinViews;
+import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsProvider;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Strings;
 import io.grpc.ManagedChannelBuilder;
 import java.io.IOException;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Logger;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -77,7 +74,10 @@ public final class BigtableDataSettings {
 
  private static final Logger LOGGER = Logger.getLogger(BigtableDataSettings.class.getName());
  private static final String BIGTABLE_EMULATOR_HOST_ENV_VAR = "BIGTABLE_EMULATOR_HOST";
- private static final AtomicBoolean BUILTIN_METRICS_REGISTERED = new AtomicBoolean(false);
+ // This is the legacy credential override used in the deprecated enableBuiltinMetrics method to
+ // override the default credentials set on the Bigtable client. Keeping it for backward
+ // compatibility.
+ @Deprecated @Nullable private static Credentials legacyMetricCredentialOverride;
 
  private final EnhancedBigtableStubSettings stubSettings;
 
@@ -197,23 +197,34 @@ public static void enableGfeOpenCensusStats() {
  com.google.cloud.bigtable.data.v2.stub.metrics.RpcViews.registerBigtableClientGfeViews();
  }
 
- /** Register built in metrics. */
- public static void enableBuiltinMetrics() throws IOException {
- if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) {
- BuiltinViews.registerBigtableBuiltinViews();
- BigtableStackdriverStatsExporter.register(GoogleCredentials.getApplicationDefault());
- }
- }
+ /**
+ * Register built in metrics.
+ *
+ * @deprecated This is a no-op that doesn't do anything. Builtin metrics are enabled by default
+ * now. Please refer to {@link
+ * BigtableDataSettings.Builder#setMetricsProvider(MetricsProvider)} on how to enable or
+ * disable built-in metrics.
+ */
+ @Deprecated
+ public static void enableBuiltinMetrics() throws IOException {}
 
  /**
  * Register built in metrics with credentials. The credentials need to have metric write access
  * for all the projects you're publishing to.
+ *
+ * @deprecated This is a no-op that doesn't do anything. Builtin metrics are enabled by default
+ * now. Please refer {@link BigtableDataSettings.Builder#setMetricsProvider(MetricsProvider)}
+ * on how to enable or disable built-in metrics.
  */
+ @Deprecated
  public static void enableBuiltinMetrics(Credentials credentials) throws IOException {
- if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) {
- BuiltinViews.registerBigtableBuiltinViews();
- BigtableStackdriverStatsExporter.register(credentials);
- }
+ BigtableDataSettings.legacyMetricCredentialOverride = credentials;
+ }
+
+ /** Get the metrics credentials if it's set by {@link #enableBuiltinMetrics(Credentials)}. */
+ @InternalApi
+ public static Credentials getMetricsCredentials() {
+ return legacyMetricCredentialOverride;
  }
 
  /** Returns the target project id. */
@@ -278,6 +289,11 @@ public boolean isBulkMutationFlowControlEnabled() {
  return stubSettings.bulkMutateRowsSettings().isServerInitiatedFlowControlEnabled();
  }
 
+ /** Gets the {@link MetricsProvider}. * */
+ public MetricsProvider getMetricsProvider() {
+ return stubSettings.getMetricsProvider();
+ }
+
  /** Returns the underlying RPC settings. */
  public EnhancedBigtableStubSettings getStubSettings() {
  return stubSettings;
@@ -527,6 +543,30 @@ public boolean isBulkMutationFlowControlEnabled() {
  return stubSettings.bulkMutateRowsSettings().isServerInitiatedFlowControlEnabled();
  }
 
+ /**
+ * Sets the {@link MetricsProvider}.
+ *
+ * <p>By default, this is set to {@link
+ * com.google.cloud.bigtable.data.v2.stub.metrics.DefaultMetricsProvider#INSTANCE} which will
+ * collect and export client side metrics.
+ *
+ * <p>To disable client side metrics, set it to {@link
+ * com.google.cloud.bigtable.data.v2.stub.metrics.NoopMetricsProvider#INSTANCE}.
+ *
+ * <p>To use a custom OpenTelemetry instance, refer to {@link
+ * com.google.cloud.bigtable.data.v2.stub.metrics.CustomOpenTelemetryMetricsProvider} on how to
+ * set it up.
+ */
+ public Builder setMetricsProvider(MetricsProvider metricsProvider) {
+ stubSettings.setMetricsProvider(metricsProvider);
+ return this;
+ }
+
+ /** Gets the {@link MetricsProvider}. */
+ public MetricsProvider getMetricsProvider() {
+ return stubSettings.getMetricsProvider();
+ }
+
  /**
  * Returns the underlying settings for making RPC calls. The settings should be changed with
  * care.

@@ -15,6 +15,10 @@
  */
 package com.google.cloud.bigtable.data.v2.stub;
 
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.APP_PROFILE_KEY;
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.INSTANCE_ID_KEY;
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.PROJECT_ID_KEY;
+
 import com.google.api.core.BetaApi;
 import com.google.api.core.InternalApi;
 import com.google.api.gax.batching.Batcher;
@@ -68,6 +72,7 @@
 import com.google.bigtable.v2.SampleRowKeysRequest;
 import com.google.bigtable.v2.SampleRowKeysResponse;
 import com.google.cloud.bigtable.Version;
+import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
 import com.google.cloud.bigtable.data.v2.internal.JwtCredentialsWithAudience;
 import com.google.cloud.bigtable.data.v2.internal.RequestContext;
 import com.google.cloud.bigtable.data.v2.models.BulkMutation;
@@ -93,8 +98,13 @@
 import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerStreamingCallable;
 import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerUnaryCallable;
 import com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsTracerFactory;
+import com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsView;
 import com.google.cloud.bigtable.data.v2.stub.metrics.CompositeTracerFactory;
+import com.google.cloud.bigtable.data.v2.stub.metrics.CustomOpenTelemetryMetricsProvider;
+import com.google.cloud.bigtable.data.v2.stub.metrics.DefaultMetricsProvider;
+import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsProvider;
 import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsTracerFactory;
+import com.google.cloud.bigtable.data.v2.stub.metrics.NoopMetricsProvider;
 import com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants;
 import com.google.cloud.bigtable.data.v2.stub.metrics.StatsHeadersServerStreamingCallable;
 import com.google.cloud.bigtable.data.v2.stub.metrics.StatsHeadersUnaryCallable;
@@ -123,6 +133,11 @@
 import io.opencensus.tags.TagValue;
 import io.opencensus.tags.Tagger;
 import io.opencensus.tags.Tags;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -241,7 +256,7 @@ public static ClientContext createClientContext(EnhancedBigtableStubSettings set
  }
 
  public static ApiTracerFactory createBigtableTracerFactory(
- EnhancedBigtableStubSettings settings, ClientContext clientContext) {
+ EnhancedBigtableStubSettings settings, ClientContext clientContext) throws IOException {
  return createBigtableTracerFactory(
  settings, Tags.getTagger(), Stats.getStatsRecorder(), clientContext);
  }
@@ -251,7 +266,8 @@ public static ApiTracerFactory createBigtableTracerFactory(
  EnhancedBigtableStubSettings settings,
  Tagger tagger,
  StatsRecorder stats,
- ClientContext clientContext) {
+ ClientContext clientContext)
+ throws IOException {
  String projectId = settings.getProjectId();
  String instanceId = settings.getInstanceId();
  String appProfileId = settings.getAppProfileId();
@@ -262,16 +278,11 @@ public static ApiTracerFactory createBigtableTracerFactory(
  .put(RpcMeasureConstants.BIGTABLE_INSTANCE_ID, TagValue.create(instanceId))
  .put(RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID, TagValue.create(appProfileId))
  .build();
- ImmutableMap<String, String> builtinAttributes =
- ImmutableMap.<String, String>builder()
- .put("project_id", projectId)
- .put("instance", instanceId)
- .put("app_profile", appProfileId)
- .build();
 
  ImmutableList.Builder<ApiTracerFactory> tracerFactories = ImmutableList.builder();
  tracerFactories
  .add(
+ // Add OpenCensus Tracing
  new OpencensusTracerFactory(
  ImmutableMap.<String, String>builder()
  // Annotate traces with the same tags as metrics
@@ -285,13 +296,47 @@ public static ApiTracerFactory createBigtableTracerFactory(
  .build()))
  // Add OpenCensus Metrics
  .add(MetricsTracerFactory.create(tagger, stats, attributes))
- .add(BuiltinMetricsTracerFactory.create(builtinAttributes))
  // Add user configured tracer
  .add(settings.getTracerFactory());
-
+ Attributes otelAttributes =
+ Attributes.of(
+ PROJECT_ID_KEY, projectId, INSTANCE_ID_KEY, instanceId, APP_PROFILE_KEY, appProfileId);
+ BuiltinMetricsTracerFactory builtinMetricsTracerFactory =
+ createBuiltinMetricsTracerFactory(
+ projectId, settings.getMetricsProvider(), otelAttributes, clientContext);
+ if (builtinMetricsTracerFactory != null) {
+ tracerFactories.add(builtinMetricsTracerFactory);
+ }
  return new CompositeTracerFactory(tracerFactories.build());
  }
 
+ private static BuiltinMetricsTracerFactory createBuiltinMetricsTracerFactory(
+ String projectId,
+ MetricsProvider metricsProvider,
+ Attributes attributes,
+ ClientContext clientContext)
+ throws IOException {
+ if (metricsProvider instanceof CustomOpenTelemetryMetricsProvider) {
+ CustomOpenTelemetryMetricsProvider customMetricsProvider =
+ (CustomOpenTelemetryMetricsProvider) metricsProvider;
+ return BuiltinMetricsTracerFactory.create(
+ customMetricsProvider.getOpenTelemetry(), attributes);
+ } else if (metricsProvider instanceof DefaultMetricsProvider) {
+ SdkMeterProviderBuilder meterProvider = SdkMeterProvider.builder();
+ Credentials credentials =
+ BigtableDataSettings.getMetricsCredentials() != null
+ ? BigtableDataSettings.getMetricsCredentials()
+ : clientContext.getCredentials();
+ BuiltinMetricsView.registerBuiltinMetrics(projectId, credentials, meterProvider);
+ OpenTelemetry openTelemetry =
+ OpenTelemetrySdk.builder().setMeterProvider(meterProvider.build()).build();
+ return BuiltinMetricsTracerFactory.create(openTelemetry, attributes);
+ } else if (metricsProvider instanceof NoopMetricsProvider) {
+ return null;
+ }
+ throw new IOException("Invalid MetricsProvider type " + metricsProvider);
+ }
+
  private static void patchCredentials(EnhancedBigtableStubSettings.Builder settings)
  throws IOException {
  int i = settings.getEndpoint().lastIndexOf(":");