Skip to content

feat: add stackdriver exporter #1247

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 30 commits into from
Aug 12, 2022
Merged

feat: add stackdriver exporter #1247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
ef67fbd
remove status from application latency
mutianf May 11, 2022
8c943fb
feat: update tracers to use built in metrics
mutianf May 10, 2022
8253a9a
feat: add response protos
mutianf Sep 8, 2021
92b1f48
feat: add response protos
mutianf Sep 8, 2021
50a1eab
feat: add stackdriver exporter
mutianf May 12, 2022
07615bd
fix tests
mutianf Jul 11, 2022
f499a1b
fix dependency
mutianf Jul 11, 2022
3ea3739
remove unused dependency
mutianf Jul 12, 2022
2032527
clean up code
mutianf Jul 12, 2022
fa1756e
udpates on comments
mutianf Jul 14, 2022
f5678b4
remove unused setting
mutianf Jul 15, 2022
1d77308
make metrics consistent with cloud monitoring
mutianf Jul 26, 2022
6c92a39
convert undefined to global
mutianf Aug 1, 2022
c3afccb
update
mutianf Aug 2, 2022
c966671
add bigtable tracer back in the base callable
mutianf Aug 2, 2022
aee2e15
fix format
mutianf Aug 2, 2022
8129f02
fix the tag name
mutianf Aug 2, 2022
551cff8
add the link to the form
mutianf Aug 3, 2022
da85a10
fix format
mutianf Aug 3, 2022
541feb1
fix dependency conflicts
mutianf Aug 4, 2022
d8f6162
fix image tests
mutianf Aug 4, 2022
b219200
update undefined cluster to global
mutianf Aug 5, 2022
a5530dd
address comments
mutianf Aug 9, 2022
8feaeec
tweak export interval
mutianf Aug 9, 2022
e960878
remove unused metric kind
mutianf Aug 10, 2022
ca63660
get project id from the metrics
mutianf Aug 11, 2022
5996119
clean up imports
mutianf Aug 11, 2022
ee04276
remove unused method and rewrite create timeseries exporter
mutianf Aug 11, 2022
7607480
fix integration test
mutianf Aug 12, 2022
559a168
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Aug 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,13 @@ implementation 'com.google.cloud:google-cloud-bigtable'
If you are using Gradle without BOM, add this to your dependencies:

```Groovy
implementation 'com.google.cloud:google-cloud-bigtable:2.10.1'
implementation 'com.google.cloud:google-cloud-bigtable:2.10.3'
```

If you are using SBT, add this to your dependencies:

```Scala
libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "2.10.1"
libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "2.10.3"
```

## Authentication
Expand Down
119 changes: 114 additions & 5 deletions google-cloud-bigtable-stats/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@
<version>2.10.2-SNAPSHOT</version><!-- {x-version-update:google-cloud-bigtable:current} -->
<description>Experimental project to shade OpenCensus dependencies.</description>

<properties>
<cloud.monitoring.version>3.3.6</cloud.monitoring.version>
</properties>

<dependencyManagement>
<dependencies>
<dependency>
Expand All @@ -25,10 +29,60 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-monitoring-bom</artifactId>
<version>${cloud.monitoring.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>

<dependencies>
<!-- opencensus dependencies -->
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-api</artifactId>
</dependency>
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-exporter-stats-stackdriver</artifactId>
</dependency>
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-impl</artifactId>
<scope>runtime</scope>
</dependency>

<!-- cloud monitoring dependencies -->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-monitoring</artifactId>
<!-- Excluding http dependencies since they're not used and are causing conflict in google-cloud-bigtable -->
<exclusions>
<exclusion>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client-gson</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>proto-google-cloud-monitoring-v3</artifactId>
</dependency>
<dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>proto-google-common-protos</artifactId>
</dependency>
<dependency>
<groupId>com.google.auth</groupId>
<artifactId>google-auth-library-credentials</artifactId>
</dependency>
<dependency>
<groupId>com.google.api</groupId>
<artifactId>gax</artifactId>
Expand All @@ -48,21 +102,35 @@
<groupId>com.google.api</groupId>
<artifactId>api-common</artifactId>
</dependency>

<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-api</artifactId>
<groupId>com.google.api</groupId>
<artifactId>gax-grpc</artifactId>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.threeten</groupId>
<artifactId>threetenbp</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>

<!-- runtime dependency for native image integration test -->
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-impl</artifactId>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client</artifactId>
<scope>runtime</scope>
</dependency>

<!-- test dependencies -->
<dependency>
<groupId>com.google.truth</groupId>
<artifactId>truth</artifactId>
Expand All @@ -73,6 +141,11 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down Expand Up @@ -111,6 +184,17 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.3.0</version>
<!-- Ignore opencensus-exporter-metrics-util and opencensus-exporter-stats-stackdriver from -->
<!-- dependency analyze. opencensus-exporter-metrics-util is a transitive dependency from -->
<!-- opencensus-exporter-stats-stackdriver but it's not defined in java-shared-dependencies. -->
<!-- To make the opencensus version consistent, only include opencensus-exporter-stats-stackdriver -->
<!-- until opencensus-exporter-metrics-util is added to java-shared-dependencies. -->
<configuration>
<ignoredDependencies>
<ignoredDependency>io.opencensus:opencensus-exporter-metrics-util:*</ignoredDependency>
<ignoredDependency>io.opencensus:opencensus-exporter-stats-stackdriver:*</ignoredDependency>
</ignoredDependencies>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
Expand All @@ -121,6 +205,31 @@
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce-version-consistency</id>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<bannedDependencies>
<includes>
<!-- gax-grpc transitvely brings in opencensus-proto, which the latest version is 0.2.0-->
<!-- Only allow 0.2.0 for opencensus-proto and then the latest version (currently 0.31.1) for all other modules-->
<!-- this will need to be updated whenever the opencensus version gets updated -->
<dependency>io.opencensus:*:[0.31.1]</dependency>
<dependency>io.opencensus:opencensus-proto:[0.2.0]</dependency>
</includes>
</bannedDependencies>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
87 changes: 87 additions & 0 deletions ...stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.cloud.bigtable.stats;

import com.google.api.MonitoredResource;
import com.google.cloud.monitoring.v3.MetricServiceClient;
import com.google.monitoring.v3.CreateTimeSeriesRequest;
import com.google.monitoring.v3.ProjectName;
import io.opencensus.exporter.metrics.util.MetricExporter;
import io.opencensus.metrics.export.Metric;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;

final class BigtableCreateTimeSeriesExporter extends MetricExporter {
private static final Logger logger =
Logger.getLogger(BigtableCreateTimeSeriesExporter.class.getName());
private final MetricServiceClient metricServiceClient;
private final MonitoredResource monitoredResource;
private final String clientId;

BigtableCreateTimeSeriesExporter(
MetricServiceClient metricServiceClient, MonitoredResource monitoredResource) {
this.metricServiceClient = metricServiceClient;
this.monitoredResource = monitoredResource;
this.clientId = BigtableStackdriverExportUtils.getDefaultTaskValue();
}

public void export(Collection<Metric> metrics) {
Map<String, List<com.google.monitoring.v3.TimeSeries>> projectToTimeSeries = new HashMap<>();

for (Metric metric : metrics) {
// only export bigtable metrics
if (!metric.getMetricDescriptor().getName().contains("bigtable")) {
continue;
}

try {
projectToTimeSeries =
metric.getTimeSeriesList().stream()
.collect(
Collectors.groupingBy(
timeSeries ->
BigtableStackdriverExportUtils.getProjectId(
metric.getMetricDescriptor(), timeSeries),
Collectors.mapping(
timeSeries ->
BigtableStackdriverExportUtils.convertTimeSeries(
metric.getMetricDescriptor(),
timeSeries,
clientId,
monitoredResource),
Collectors.toList())));

for (Map.Entry<String, List<com.google.monitoring.v3.TimeSeries>> entry :
projectToTimeSeries.entrySet()) {
ProjectName projectName = ProjectName.of(entry.getKey());
CreateTimeSeriesRequest request =
CreateTimeSeriesRequest.newBuilder()
.setName(projectName.toString())
.addAllTimeSeries(entry.getValue())
.build();
this.metricServiceClient.createServiceTimeSeries(request);
}
} catch (Throwable e) {
logger.log(Level.WARNING, "Exception thrown when exporting TimeSeries.", e);
}
}
}
}
Loading