feat: migrate per connection error count metric to otel (#2133)

* feat: migrate per connection error count metric to otel * update test * address comments * remove unnecessary check * clean up statsRecorder * remove dependency * address comments
googleapis · mutianf · Apr 12, 2024 · Jan 23, 2024 · Feb 5, 2024 · Feb 16, 2024
commit cd3467c23aeb35400eb639f8014ceabe92ee01fe
@@ -186,4 +186,10 @@
  <className>com/google/cloud/bigtable/data/v2/models/MutateRowsException</className>
  <method>*</method>
  </difference>
+ <!-- InternalApi was updated -->
+ <difference>
+ <differenceType>7004</differenceType>
+ <className>com/google/cloud/bigtable/data/v2/stub/metrics/ErrorCountPerConnectionMetricTracker</className>
+ <method>*</method>
+ </difference>
 </differences>
@@ -19,6 +19,7 @@
 import com.google.api.gax.core.BackgroundResource;
 import com.google.api.gax.rpc.ClientContext;
 import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStub;
+import io.opentelemetry.api.OpenTelemetry;
 import java.io.IOException;
 import javax.annotation.Nonnull;
 
@@ -64,6 +65,7 @@
 public final class BigtableDataClientFactory implements AutoCloseable {
  private final BigtableDataSettings defaultSettings;
  private final ClientContext sharedClientContext;
+ private final OpenTelemetry openTelemetry;
 
  /**
  * Create a instance of this factory.
@@ -75,13 +77,21 @@ public static BigtableDataClientFactory create(BigtableDataSettings defaultSetti
  throws IOException {
  ClientContext sharedClientContext =
  EnhancedBigtableStub.createClientContext(defaultSettings.getStubSettings());
- return new BigtableDataClientFactory(sharedClientContext, defaultSettings);
+ OpenTelemetry openTelemetry =
+ EnhancedBigtableStub.getOpenTelemetry(
+ defaultSettings.getProjectId(),
+ defaultSettings.getMetricsProvider(),
+ sharedClientContext.getCredentials());
+ return new BigtableDataClientFactory(sharedClientContext, defaultSettings, openTelemetry);
  }
 
  private BigtableDataClientFactory(
- ClientContext sharedClientContext, BigtableDataSettings defaultSettings) {
+ ClientContext sharedClientContext,
+ BigtableDataSettings defaultSettings,
+ OpenTelemetry openTelemetry) {
  this.sharedClientContext = sharedClientContext;
  this.defaultSettings = defaultSettings;
+ this.openTelemetry = openTelemetry;
  }
 
  /**
@@ -112,7 +122,7 @@ public BigtableDataClient createDefault() {
  .toBuilder()
  .setTracerFactory(
  EnhancedBigtableStub.createBigtableTracerFactory(
- defaultSettings.getStubSettings(), sharedClientContext))
+ defaultSettings.getStubSettings(), openTelemetry))
  .build();
 
  return BigtableDataClient.createWithClientContext(defaultSettings, clientContext);
@@ -141,7 +151,7 @@ public BigtableDataClient createForAppProfile(@Nonnull String appProfileId) thro
  .toBuilder()
  .setTracerFactory(
  EnhancedBigtableStub.createBigtableTracerFactory(
- settings.getStubSettings(), sharedClientContext))
+ settings.getStubSettings(), openTelemetry))
  .build();
  return BigtableDataClient.createWithClientContext(settings, clientContext);
  }
@@ -170,7 +180,7 @@ public BigtableDataClient createForInstance(@Nonnull String projectId, @Nonnull
  .toBuilder()
  .setTracerFactory(
  EnhancedBigtableStub.createBigtableTracerFactory(
- settings.getStubSettings(), sharedClientContext))
+ settings.getStubSettings(), openTelemetry))
  .build();
 
  return BigtableDataClient.createWithClientContext(settings, clientContext);
@@ -200,7 +210,7 @@ public BigtableDataClient createForInstance(
  .toBuilder()
  .setTracerFactory(
  EnhancedBigtableStub.createBigtableTracerFactory(
- settings.getStubSettings(), sharedClientContext))
+ settings.getStubSettings(), openTelemetry))
  .build();
  return BigtableDataClient.createWithClientContext(settings, clientContext);
  }

@@ -16,6 +16,7 @@
 package com.google.cloud.bigtable.data.v2.stub;
 
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.APP_PROFILE_KEY;
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.CLIENT_NAME_KEY;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.INSTANCE_ID_KEY;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.PROJECT_ID_KEY;
 
@@ -99,12 +100,11 @@
 import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerStreamingCallable;
 import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerUnaryCallable;
 import com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsTracerFactory;
-import com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsView;
 import com.google.cloud.bigtable.data.v2.stub.metrics.CompositeTracerFactory;
 import com.google.cloud.bigtable.data.v2.stub.metrics.CustomOpenTelemetryMetricsProvider;
 import com.google.cloud.bigtable.data.v2.stub.metrics.DefaultMetricsProvider;
-import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsProvider;
 import com.google.cloud.bigtable.data.v2.stub.metrics.ErrorCountPerConnectionMetricTracker;
+import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsProvider;
 import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsTracerFactory;
 import com.google.cloud.bigtable.data.v2.stub.metrics.NoopMetricsProvider;
 import com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants;
@@ -138,9 +138,6 @@
 import io.opencensus.tags.Tags;
 import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.common.Attributes;
-import io.opentelemetry.sdk.OpenTelemetrySdk;
-import io.opentelemetry.sdk.metrics.SdkMeterProvider;
-import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -194,10 +191,13 @@ public class EnhancedBigtableStub implements AutoCloseable {
  public static EnhancedBigtableStub create(EnhancedBigtableStubSettings settings)
  throws IOException {
  ClientContext clientContext = createClientContext(settings);
+ OpenTelemetry openTelemetry =
+ getOpenTelemetry(
+ settings.getProjectId(), settings.getMetricsProvider(), clientContext.getCredentials());
  ClientContext contextWithTracer =
  clientContext
  .toBuilder()
- .setTracerFactory(createBigtableTracerFactory(settings, clientContext))
+ .setTracerFactory(createBigtableTracerFactory(settings, openTelemetry))
  .build();
 
  return new EnhancedBigtableStub(settings, contextWithTracer);
@@ -219,15 +219,26 @@ public static ClientContext createClientContext(EnhancedBigtableStubSettings set
  // workaround JWT audience issues
  patchCredentials(builder);
 
+ // Fix the credentials so that they can be shared
+ Credentials credentials = null;
+ if (builder.getCredentialsProvider() != null) {
+ credentials = builder.getCredentialsProvider().getCredentials();
+ }
+ builder.setCredentialsProvider(FixedCredentialsProvider.create(credentials));
+
  InstantiatingGrpcChannelProvider.Builder transportProvider =
  builder.getTransportChannelProvider() instanceof InstantiatingGrpcChannelProvider
  ? ((InstantiatingGrpcChannelProvider) builder.getTransportChannelProvider()).toBuilder()
  : null;
 
+ OpenTelemetry openTelemetry =
+ getOpenTelemetry(settings.getProjectId(), settings.getMetricsProvider(), credentials);
  ErrorCountPerConnectionMetricTracker errorCountPerConnectionMetricTracker;
- if (transportProvider != null) {
+ // Skip setting up ErrorCountPerConnectionMetricTracker if openTelemetry is null
+ if (openTelemetry != null && transportProvider != null) {
  errorCountPerConnectionMetricTracker =
- new ErrorCountPerConnectionMetricTracker(createBuiltinAttributes(builder));
+ new ErrorCountPerConnectionMetricTracker(
+ openTelemetry, createBuiltinAttributes(settings));
  ApiFunction<ManagedChannelBuilder, ManagedChannelBuilder> oldChannelConfigurator =
  transportProvider.getChannelConfigurator();
  transportProvider.setChannelConfigurator(
@@ -249,12 +260,6 @@ public static ClientContext createClientContext(EnhancedBigtableStubSettings set
 
  // Inject channel priming
  if (settings.isRefreshingChannel()) {
- // Fix the credentials so that they can be shared
- Credentials credentials = null;
- if (builder.getCredentialsProvider() != null) {
- credentials = builder.getCredentialsProvider().getCredentials();
- }
- builder.setCredentialsProvider(FixedCredentialsProvider.create(credentials));
 
  if (transportProvider != null) {
  transportProvider.setChannelPrimer(
@@ -279,17 +284,17 @@ public static ClientContext createClientContext(EnhancedBigtableStubSettings set
  }
 
  public static ApiTracerFactory createBigtableTracerFactory(
- EnhancedBigtableStubSettings settings, ClientContext clientContext) throws IOException {
+ EnhancedBigtableStubSettings settings, OpenTelemetry openTelemetry) throws IOException {
  return createBigtableTracerFactory(
- settings, Tags.getTagger(), Stats.getStatsRecorder(), clientContext);
+ settings, Tags.getTagger(), Stats.getStatsRecorder(), openTelemetry);
  }
 
  @VisibleForTesting
  public static ApiTracerFactory createBigtableTracerFactory(
  EnhancedBigtableStubSettings settings,
  Tagger tagger,
  StatsRecorder stats,
- ClientContext clientContext)
+ OpenTelemetry openTelemetry)
  throws IOException {
  String projectId = settings.getProjectId();
  String instanceId = settings.getInstanceId();
@@ -302,8 +307,6 @@ public static ApiTracerFactory createBigtableTracerFactory(
  .put(RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID, TagValue.create(appProfileId))
  .build();
 
- ImmutableMap<String, String> builtinAttributes = createBuiltinAttributes(settings.toBuilder());
-
  ImmutableList.Builder<ApiTracerFactory> tracerFactories = ImmutableList.builder();
  tracerFactories
  .add(
@@ -323,53 +326,47 @@ public static ApiTracerFactory createBigtableTracerFactory(
  .add(MetricsTracerFactory.create(tagger, stats, attributes))
  // Add user configured tracer
  .add(settings.getTracerFactory());
- Attributes otelAttributes =
- Attributes.of(
- PROJECT_ID_KEY, projectId, INSTANCE_ID_KEY, instanceId, APP_PROFILE_KEY, appProfileId);
  BuiltinMetricsTracerFactory builtinMetricsTracerFactory =
- createBuiltinMetricsTracerFactory(
- projectId, settings.getMetricsProvider(), otelAttributes, clientContext);
+ openTelemetry != null
+ ? BuiltinMetricsTracerFactory.create(openTelemetry, createBuiltinAttributes(settings))
+ : null;
  if (builtinMetricsTracerFactory != null) {
  tracerFactories.add(builtinMetricsTracerFactory);
  }
  return new CompositeTracerFactory(tracerFactories.build());
  }
 
- private static BuiltinMetricsTracerFactory createBuiltinMetricsTracerFactory(
- String projectId,
- MetricsProvider metricsProvider,
- Attributes attributes,
- ClientContext clientContext)
+ @Nullable
+ public static OpenTelemetry getOpenTelemetry(
+ String projectId, MetricsProvider metricsProvider, @Nullable Credentials defaultCredentials)
  throws IOException {
  if (metricsProvider instanceof CustomOpenTelemetryMetricsProvider) {
  CustomOpenTelemetryMetricsProvider customMetricsProvider =
  (CustomOpenTelemetryMetricsProvider) metricsProvider;
- return BuiltinMetricsTracerFactory.create(
- customMetricsProvider.getOpenTelemetry(), attributes);
+ return customMetricsProvider.getOpenTelemetry();
  } else if (metricsProvider instanceof DefaultMetricsProvider) {
- SdkMeterProviderBuilder meterProvider = SdkMeterProvider.builder();
  Credentials credentials =
  BigtableDataSettings.getMetricsCredentials() != null
  ? BigtableDataSettings.getMetricsCredentials()
- : clientContext.getCredentials();
- BuiltinMetricsView.registerBuiltinMetrics(projectId, credentials, meterProvider);
- OpenTelemetry openTelemetry =
- OpenTelemetrySdk.builder().setMeterProvider(meterProvider.build()).build();
- return BuiltinMetricsTracerFactory.create(openTelemetry, attributes);
+ : defaultCredentials;
+ DefaultMetricsProvider defaultMetricsProvider = (DefaultMetricsProvider) metricsProvider;
+ return defaultMetricsProvider.getOpenTelemetry(projectId, credentials);
  } else if (metricsProvider instanceof NoopMetricsProvider) {
  return null;
  }
  throw new IOException("Invalid MetricsProvider type " + metricsProvider);
  }
 
- private static ImmutableMap<String, String> createBuiltinAttributes(
- EnhancedBigtableStubSettings.Builder builder) {
- return ImmutableMap.<String, String>builder()
- .put("project_id", builder.getProjectId())
- .put("instance", builder.getInstanceId())
- .put("app_profile", builder.getAppProfileId())
- .put("client_name", "bigtable-java/" + Version.VERSION)
- .build();
+ private static Attributes createBuiltinAttributes(EnhancedBigtableStubSettings settings) {
+ return Attributes.of(
+ PROJECT_ID_KEY,
+ settings.getProjectId(),
+ INSTANCE_ID_KEY,
+ settings.getInstanceId(),
+ APP_PROFILE_KEY,
+ settings.getAppProfileId(),
+ CLIENT_NAME_KEY,
+ "bigtable-java/" + Version.VERSION);
  }
 
  private static void patchCredentials(EnhancedBigtableStubSettings.Builder settings)

@@ -20,7 +20,9 @@
 import com.google.api.core.ApiFutureCallback;
 import com.google.api.core.ApiFutures;
 import com.google.api.core.InternalApi;
+import com.google.api.gax.core.CredentialsProvider;
 import com.google.api.gax.core.FixedCredentialsProvider;
+import com.google.api.gax.core.NoCredentialsProvider;
 import com.google.auth.Credentials;
 import com.google.cloud.monitoring.v3.MetricServiceClient;
 import com.google.cloud.monitoring.v3.MetricServiceSettings;
@@ -43,6 +45,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import javax.annotation.Nullable;
 import org.threeten.bp.Duration;
 
 /**
@@ -75,10 +78,16 @@ public final class BigtableCloudMonitoringExporter implements MetricExporter {
 
  private CompletableResultCode lastExportCode;
 
- public static BigtableCloudMonitoringExporter create(String projectId, Credentials credentials)
- throws IOException {
+ public static BigtableCloudMonitoringExporter create(
+ String projectId, @Nullable Credentials credentials) throws IOException {
  MetricServiceSettings.Builder settingsBuilder = MetricServiceSettings.newBuilder();
- settingsBuilder.setCredentialsProvider(FixedCredentialsProvider.create(credentials));
+ CredentialsProvider credentialsProvider;
+ if (credentials == null) {
+ credentialsProvider = NoCredentialsProvider.create();
+ } else {
+ credentialsProvider = FixedCredentialsProvider.create(credentials);
+ }
+ settingsBuilder.setCredentialsProvider(credentialsProvider);
  settingsBuilder.setEndpoint(MONITORING_ENDPOINT);
 
  org.threeten.bp.Duration timeout = Duration.ofMinutes(1);