Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: Fix export to log detect resource errors #2197

Merged
merged 4 commits into from
Apr 12, 2024
Merged

fix: Fix export to log detect resource errors #2197

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ecabc86
fix: Fix export to log detect resource errors
mutianf Apr 12, 2024
a1e3b29
catch exceptions
mutianf Apr 12, 2024
7cdfde6
update error handling
mutianf Apr 12, 2024
6148b64
remove imports
mutianf Apr 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Prev Previous commit
Next Next commit
update error handling
  • Loading branch information
@mutianf
mutianf committed Apr 12, 2024
commit 7cdfde670fa327bf0a60352a5b17e6c64e78ec26
22 changes: 18 additions & 4 deletions ...d-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java
View file
Open in desktop
Expand Up @@ -149,6 +149,8 @@
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;

Expand All @@ -166,6 +168,9 @@
*/
@InternalApi
public class EnhancedBigtableStub implements AutoCloseable {

private static final Logger logger = Logger.getLogger(EnhancedBigtableStub.class.getName());

private static final String CLIENT_NAME = "Bigtable";
private static final long FLOW_CONTROL_ADJUSTING_INTERVAL_MS = TimeUnit.SECONDS.toMillis(20);
private final EnhancedBigtableStubSettings settings;
Expand Down Expand Up @@ -238,8 +243,16 @@ public static ClientContext createClientContext(EnhancedBigtableStubSettings set
? ((InstantiatingGrpcChannelProvider) builder.getTransportChannelProvider()).toBuilder()
: null;

OpenTelemetry openTelemetry =
getOpenTelemetry(settings.getProjectId(), settings.getMetricsProvider(), credentials);
OpenTelemetry openTelemetry = null;
try {
// We don't want client side metrics to crash the client, so catch any exception when getting
// the OTEL instance
// and log the exception instead.
openTelemetry =
getOpenTelemetry(settings.getProjectId(), settings.getMetricsProvider(), credentials);
} catch (Throwable t) {
logger.log(Level.WARNING, "Failed to get OTEL, will skip exporting client side metrics", t);
}
ErrorCountPerConnectionMetricTracker errorCountPerConnectionMetricTracker;
// Skip setting up ErrorCountPerConnectionMetricTracker if openTelemetry is null
if (openTelemetry != null && transportProvider != null) {
Expand Down Expand Up @@ -291,7 +304,8 @@ public static ClientContext createClientContext(EnhancedBigtableStubSettings set
}

public static ApiTracerFactory createBigtableTracerFactory(
EnhancedBigtableStubSettings settings, OpenTelemetry openTelemetry) throws IOException {
EnhancedBigtableStubSettings settings, @Nullable OpenTelemetry openTelemetry)
throws IOException {
return createBigtableTracerFactory(
settings, Tags.getTagger(), Stats.getStatsRecorder(), openTelemetry);
}
Expand All @@ -301,7 +315,7 @@ public static ApiTracerFactory createBigtableTracerFactory(
EnhancedBigtableStubSettings settings,
Tagger tagger,
StatsRecorder stats,
OpenTelemetry openTelemetry)
@Nullable OpenTelemetry openTelemetry)
throws IOException {
String projectId = settings.getProjectId();
String instanceId = settings.getInstanceId();
Expand Down
236 changes: 113 additions & 123 deletions .../java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java
View file
Open in desktop
Expand Up @@ -172,153 +172,143 @@ public CompletableResultCode export(Collection<MetricData> collection) {

/** Export metrics associated with a BigtableTable resource. */
private CompletableResultCode exportBigtableResourceMetrics(Collection<MetricData> collection) {
// Filter bigtable table metrics
List<MetricData> bigtableMetricData =
collection.stream()
.filter(md -> BIGTABLE_TABLE_METRICS.contains(md.getName()))
.collect(Collectors.toList());

// Skips exporting if there's none
if (bigtableMetricData.isEmpty()) {
return CompletableResultCode.ofSuccess();
}

// Verifies metrics project id are the same as the bigtable project id set on this client
if (!bigtableMetricData.stream()
.flatMap(metricData -> metricData.getData().getPoints().stream())
.allMatch(pd -> bigtableProjectId.equals(BigtableExporterUtils.getProjectId(pd)))) {
logger.log(Level.WARNING, "Metric data has different a projectId. Skip exporting.");
return CompletableResultCode.ofFailure();
}

List<TimeSeries> bigtableTimeSeries;
try {
// Filter bigtable table metrics
List<MetricData> bigtableMetricData =
collection.stream()
.filter(md -> BIGTABLE_TABLE_METRICS.contains(md.getName()))
.collect(Collectors.toList());

// Skips exporting if there's none
if (bigtableMetricData.isEmpty()) {
return CompletableResultCode.ofSuccess();
}

// Verifies metrics project id are the same as the bigtable project id set on this client
if (!bigtableMetricData.stream()
.flatMap(metricData -> metricData.getData().getPoints().stream())
.allMatch(pd -> bigtableProjectId.equals(BigtableExporterUtils.getProjectId(pd)))) {
logger.log(Level.WARNING, "Metric data has different a projectId. Skip exporting.");
return CompletableResultCode.ofFailure();
}

List<TimeSeries> bigtableTimeSeries;
try {
bigtableTimeSeries =
BigtableExporterUtils.convertToBigtableTimeSeries(bigtableMetricData, taskId);
} catch (Throwable e) {
logger.log(
Level.WARNING,
"Failed to convert bigtable table metric data to cloud monitoring timeseries.",
e);
return CompletableResultCode.ofFailure();
}

ProjectName projectName = ProjectName.of(bigtableProjectId);
CreateTimeSeriesRequest bigtableRequest =
bigtableTimeSeries =
BigtableExporterUtils.convertToBigtableTimeSeries(bigtableMetricData, taskId);
} catch (Throwable e) {
logger.log(
Level.WARNING,
"Failed to convert bigtable table metric data to cloud monitoring timeseries.",
e);
return CompletableResultCode.ofFailure();
}

ProjectName projectName = ProjectName.of(bigtableProjectId);
CreateTimeSeriesRequest bigtableRequest =
CreateTimeSeriesRequest.newBuilder()
.setName(projectName.toString())
.addAllTimeSeries(bigtableTimeSeries)
.build();

ApiFuture<Empty> future =
this.client.createServiceTimeSeriesCallable().futureCall(bigtableRequest);

CompletableResultCode bigtableExportCode = new CompletableResultCode();
ApiFutures.addCallback(
future,
new ApiFutureCallback<Empty>() {
@Override
public void onFailure(Throwable throwable) {
logger.log(
Level.WARNING,
"createServiceTimeSeries request failed for bigtable metrics. ",
throwable);
bigtableExportCode.fail();
}

@Override
public void onSuccess(Empty empty) {
bigtableExportCode.succeed();
}
},
MoreExecutors.directExecutor());

return bigtableExportCode;
}

/** Export metrics associated with the resource the Application is running on. */
private CompletableResultCode exportApplicationResourceMetrics(
Collection<MetricData> collection) {
if (applicationResource == null) {
return CompletableResultCode.ofSuccess();
}

// Filter application level metrics
List<MetricData> metricData =
collection.stream()
.filter(md -> APPLICATION_METRICS.contains(md.getName()))
.collect(Collectors.toList());

// Skip exporting if there's none
if (metricData.isEmpty()) {
return CompletableResultCode.ofSuccess();
}

List<TimeSeries> timeSeries;
try {
timeSeries =
BigtableExporterUtils.convertToApplicationResourceTimeSeries(
metricData, taskId, applicationResource);
} catch (Throwable e) {
logger.log(
Level.WARNING,
"Failed to convert application metric data to cloud monitoring timeseries.",
e);
return CompletableResultCode.ofFailure();
}

// Construct the request. The project id will be the project id of the detected monitored
// resource.
ApiFuture<Empty> gceOrGkeFuture;
CompletableResultCode exportCode = new CompletableResultCode();
try {
ProjectName projectName =
ProjectName.of(applicationResource.getLabelsOrThrow(APPLICATION_RESOURCE_PROJECT_ID));
CreateTimeSeriesRequest request =
CreateTimeSeriesRequest.newBuilder()
.setName(projectName.toString())
.addAllTimeSeries(bigtableTimeSeries)
.addAllTimeSeries(timeSeries)
.build();

ApiFuture<Empty> future =
this.client.createServiceTimeSeriesCallable().futureCall(bigtableRequest);
gceOrGkeFuture = this.client.createServiceTimeSeriesCallable().futureCall(request);

CompletableResultCode bigtableExportCode = new CompletableResultCode();
ApiFutures.addCallback(
future,
gceOrGkeFuture,
new ApiFutureCallback<Empty>() {
@Override
public void onFailure(Throwable throwable) {
logger.log(
Level.WARNING,
"createServiceTimeSeries request failed for bigtable metrics. ",
"createServiceTimeSeries request failed for per connection error metrics.",
throwable);
bigtableExportCode.fail();
exportCode.fail();
}

@Override
public void onSuccess(Empty empty) {
bigtableExportCode.succeed();
exportCode.succeed();
}
},
MoreExecutors.directExecutor());

return bigtableExportCode;
} catch (Throwable t) {
logger.log(Level.WARNING, "Failed to export Bigtable Resource Metrics.", t);
} catch (Exception e) {
logger.log(
Level.WARNING,
"Failed to get projectName for application resource " + applicationResource);
return CompletableResultCode.ofFailure();
}
}

/** Export metrics associated with the resource the Application is running on. */
private CompletableResultCode exportApplicationResourceMetrics(
Collection<MetricData> collection) {
try {
if (applicationResource == null) {
return CompletableResultCode.ofSuccess();
}

// Filter application level metrics
List<MetricData> metricData =
collection.stream()
.filter(md -> APPLICATION_METRICS.contains(md.getName()))
.collect(Collectors.toList());

// Skip exporting if there's none
if (metricData.isEmpty()) {
return CompletableResultCode.ofSuccess();
}

List<TimeSeries> timeSeries;
try {
timeSeries =
BigtableExporterUtils.convertToApplicationResourceTimeSeries(
metricData, taskId, applicationResource);
} catch (Throwable e) {
logger.log(
Level.WARNING,
"Failed to convert application metric data to cloud monitoring timeseries.",
e);
return CompletableResultCode.ofFailure();
}

// Construct the request. The project id will be the project id of the detected monitored
// resource.
ApiFuture<Empty> gceOrGkeFuture;
CompletableResultCode exportCode = new CompletableResultCode();
try {
ProjectName projectName =
ProjectName.of(applicationResource.getLabelsOrThrow(APPLICATION_RESOURCE_PROJECT_ID));
CreateTimeSeriesRequest request =
CreateTimeSeriesRequest.newBuilder()
.setName(projectName.toString())
.addAllTimeSeries(timeSeries)
.build();

gceOrGkeFuture = this.client.createServiceTimeSeriesCallable().futureCall(request);

ApiFutures.addCallback(
gceOrGkeFuture,
new ApiFutureCallback<Empty>() {
@Override
public void onFailure(Throwable throwable) {
logger.log(
Level.WARNING,
"createServiceTimeSeries request failed for per connection error metrics.",
throwable);
exportCode.fail();
}

@Override
public void onSuccess(Empty empty) {
exportCode.succeed();
}
},
MoreExecutors.directExecutor());

} catch (Exception e) {
logger.log(
Level.WARNING,
"Failed to get projectName for application resource " + applicationResource);
return CompletableResultCode.ofFailure();
}

return exportCode;
} catch (Throwable t) {
logger.log(Level.WARNING, "Failed to export Application Resource Metrics.", t);
return CompletableResultCode.ofFailure();
}
return exportCode;
}

@Override
Expand Down
5 changes: 4 additions & 1 deletion ...e/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableExporterUtils.java
View file
Open in desktop
Expand Up @@ -166,7 +166,10 @@ static MonitoredResource detectResource() {
break;
}
} catch (IllegalStateException e) {
logger.log(Level.WARNING, "Failed to create monitored resource for GCE / GKE.", e);
logger.log(
Level.WARNING,
"Failed to create monitored resource for " + detectedPlatform.getSupportedPlatform(),
e);
}
return monitoredResource;
}
Expand Down
16 changes: 5 additions & 11 deletions ...able/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsView.java
View file
Open in desktop
Expand Up @@ -24,7 +24,6 @@
import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;
import java.io.IOException;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.annotation.Nullable;

Expand Down Expand Up @@ -53,16 +52,11 @@ public static void registerBuiltinMetrics(String projectId, SdkMeterProviderBuil
public static void registerBuiltinMetrics(
String projectId, @Nullable Credentials credentials, SdkMeterProviderBuilder builder)
throws IOException {
try {
MetricExporter metricExporter =
BigtableCloudMonitoringExporter.create(projectId, credentials);
for (Map.Entry<InstrumentSelector, View> entry :
BuiltinMetricsConstants.getAllViews().entrySet()) {
builder.registerView(entry.getKey(), entry.getValue());
}
builder.registerMetricReader(PeriodicMetricReader.create(metricExporter));
} catch (Throwable t) {
logger.log(Level.WARNING, "Failed to register builtin metrics.", t);
MetricExporter metricExporter = BigtableCloudMonitoringExporter.create(projectId, credentials);
for (Map.Entry<InstrumentSelector, View> entry :
BuiltinMetricsConstants.getAllViews().entrySet()) {
builder.registerView(entry.getKey(), entry.getValue());
}
builder.registerMetricReader(PeriodicMetricReader.create(metricExporter));
}
}