From d32fbb78bbde2ad04103ab7b2c1176a6df72d0a3 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Fri, 12 Apr 2024 18:12:16 -0400 Subject: [PATCH] fix: Fix export to log detect resource errors (#2197) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://togithub.com/googleapis/java-bigtable/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) - [ ] Rollback plan is reviewed and LGTMed - [ ] All new data plane features have a completed end to end testing plan Fixes # ☕️ If you write sample code, please follow the [samples format]( https://togithub.com/GoogleCloudPlatform/java-docs-samples/blob/main/SAMPLE_FORMAT.md). --- .../data/v2/stub/EnhancedBigtableStub.java | 21 +++++++-- .../stub/metrics/BigtableExporterUtils.java | 46 +++++++++++++------ 2 files changed, 50 insertions(+), 17 deletions(-) diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index f0aa85233..5dbb59aed 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -149,6 +149,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -166,6 +168,9 @@ */ @InternalApi public class EnhancedBigtableStub implements AutoCloseable { + + private static final Logger logger = Logger.getLogger(EnhancedBigtableStub.class.getName()); + private static final String CLIENT_NAME = "Bigtable"; private static final long FLOW_CONTROL_ADJUSTING_INTERVAL_MS = TimeUnit.SECONDS.toMillis(20); private final EnhancedBigtableStubSettings settings; @@ -238,8 +243,15 @@ public static ClientContext createClientContext(EnhancedBigtableStubSettings set ? ((InstantiatingGrpcChannelProvider) builder.getTransportChannelProvider()).toBuilder() : null; - OpenTelemetry openTelemetry = - getOpenTelemetry(settings.getProjectId(), settings.getMetricsProvider(), credentials); + OpenTelemetry openTelemetry = null; + try { + // We don't want client side metrics to crash the client, so catch any exception when getting + // the OTEL instance and log the exception instead. + openTelemetry = + getOpenTelemetry(settings.getProjectId(), settings.getMetricsProvider(), credentials); + } catch (Throwable t) { + logger.log(Level.WARNING, "Failed to get OTEL, will skip exporting client side metrics", t); + } ErrorCountPerConnectionMetricTracker errorCountPerConnectionMetricTracker; // Skip setting up ErrorCountPerConnectionMetricTracker if openTelemetry is null if (openTelemetry != null && transportProvider != null) { @@ -291,7 +303,8 @@ public static ClientContext createClientContext(EnhancedBigtableStubSettings set } public static ApiTracerFactory createBigtableTracerFactory( - EnhancedBigtableStubSettings settings, OpenTelemetry openTelemetry) throws IOException { + EnhancedBigtableStubSettings settings, @Nullable OpenTelemetry openTelemetry) + throws IOException { return createBigtableTracerFactory( settings, Tags.getTagger(), Stats.getStatsRecorder(), openTelemetry); } @@ -301,7 +314,7 @@ public static ApiTracerFactory createBigtableTracerFactory( EnhancedBigtableStubSettings settings, Tagger tagger, StatsRecorder stats, - OpenTelemetry openTelemetry) + @Nullable OpenTelemetry openTelemetry) throws IOException { String projectId = settings.getProjectId(); String instanceId = settings.getInstanceId(); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableExporterUtils.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableExporterUtils.java index 9a4d928ce..5bf6688e1 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableExporterUtils.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableExporterUtils.java @@ -151,16 +151,27 @@ static List convertToApplicationResourceTimeSeries( static MonitoredResource detectResource() { GCPPlatformDetector detector = GCPPlatformDetector.DEFAULT_INSTANCE; DetectedPlatform detectedPlatform = detector.detectPlatform(); - switch (detectedPlatform.getSupportedPlatform()) { - case GOOGLE_COMPUTE_ENGINE: - return createGceMonitoredResource( - detectedPlatform.getProjectId(), detectedPlatform.getAttributes()); - case GOOGLE_KUBERNETES_ENGINE: - return createGkeMonitoredResource( - detectedPlatform.getProjectId(), detectedPlatform.getAttributes()); - default: - return null; + MonitoredResource monitoredResource = null; + try { + switch (detectedPlatform.getSupportedPlatform()) { + case GOOGLE_COMPUTE_ENGINE: + monitoredResource = + createGceMonitoredResource( + detectedPlatform.getProjectId(), detectedPlatform.getAttributes()); + break; + case GOOGLE_KUBERNETES_ENGINE: + monitoredResource = + createGkeMonitoredResource( + detectedPlatform.getProjectId(), detectedPlatform.getAttributes()); + break; + } + } catch (IllegalStateException e) { + logger.log( + Level.WARNING, + "Failed to create monitored resource for " + detectedPlatform.getSupportedPlatform(), + e); } + return monitoredResource; } private static MonitoredResource createGceMonitoredResource( @@ -168,8 +179,8 @@ private static MonitoredResource createGceMonitoredResource( return MonitoredResource.newBuilder() .setType("gce_instance") .putLabels("project_id", projectId) - .putLabels("instance_id", attributes.get(AttributeKeys.GCE_INSTANCE_ID)) - .putLabels("zone", attributes.get(AttributeKeys.GCE_AVAILABILITY_ZONE)) + .putLabels("instance_id", getAttribute(attributes, AttributeKeys.GCE_INSTANCE_ID)) + .putLabels("zone", getAttribute(attributes, AttributeKeys.GCE_AVAILABILITY_ZONE)) .build(); } @@ -178,14 +189,23 @@ private static MonitoredResource createGkeMonitoredResource( return MonitoredResource.newBuilder() .setType("k8s_container") .putLabels("project_id", projectId) - .putLabels("location", attributes.get(AttributeKeys.GKE_CLUSTER_LOCATION)) - .putLabels("cluster_name", attributes.get(AttributeKeys.GKE_CLUSTER_NAME)) + .putLabels("location", getAttribute(attributes, AttributeKeys.GKE_CLUSTER_LOCATION)) + .putLabels("cluster_name", getAttribute(attributes, AttributeKeys.GKE_CLUSTER_NAME)) .putLabels("namespace_name", MoreObjects.firstNonNull(System.getenv("NAMESPACE"), "")) .putLabels("pod_name", MoreObjects.firstNonNull(System.getenv("HOSTNAME"), "")) .putLabels("container_name", MoreObjects.firstNonNull(System.getenv("CONTAINER_NAME"), "")) .build(); } + private static String getAttribute(Map attributes, String key) { + String value = attributes.get(key); + if (value == null) { + throw new IllegalStateException( + "Required attribute " + key + " does not exist in the attributes map " + attributes); + } + return value; + } + private static TimeSeries convertPointToBigtableTimeSeries( MetricData metricData, PointData pointData, String taskId) { TimeSeries.Builder builder =