Skip to content

Commit 9a9cd38

Browse files
david-leifkerdavid-leifker
authored
fix(open-telemetry): include missing dependency (#12882)
1 parent c9d77fd commit 9a9cd38

File tree

13 files changed

+126
-7
lines changed

13 files changed

+126
-7
lines changed

‎build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ project.ext.externalDependency = [
227227
'opentelemetrySdk': 'io.opentelemetry:opentelemetry-sdk:' + openTelemetryVersion,
228228
'opentelemetrySdkTrace': 'io.opentelemetry:opentelemetry-sdk-trace:' + openTelemetryVersion,
229229
'opentelemetryAutoConfig': 'io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:' + openTelemetryVersion,
230+
'opentelemetryExporter': 'io.opentelemetry:opentelemetry-exporter-otlp:' + openTelemetryVersion,
230231
'opentelemetryAnnotations': 'io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations:2.11.0',
231232
'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15',
232233
'parquet': 'org.apache.parquet:parquet-avro:1.12.3',

‎datahub-frontend/play.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ dependencies {
9494
implementation externalDependency.slf4jApi
9595
compileOnly externalDependency.lombok
9696
runtimeOnly externalDependency.guicePlay
97+
runtimeOnly externalDependency.opentelemetryExporter
9798
runtimeOnly (externalDependency.playDocs) {
9899
exclude group: 'com.typesafe.akka', module: "akka-http-core_$playScalaVersion"
99100
}

‎datahub-upgrade/build.gradle

+2
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ dependencies {
7171
implementation externalDependency.springBootAutoconfigure
7272
implementation externalDependency.springCore
7373
implementation externalDependency.springKafka
74+
runtimeOnly externalDependency.opentelemetryExporter
7475

7576
runtimeOnly externalDependency.logbackClassic
7677
runtimeOnly externalDependency.mariadbConnector
@@ -174,6 +175,7 @@ docker {
174175
files bootJar.outputs.files
175176
files fileTree(rootProject.projectDir) {
176177
include '.dockerignore'
178+
include 'docker/monitoring/*'
177179
include "docker/${docker_repo}/*"
178180
include 'metadata-models/src/main/resources/*'
179181
}.exclude {

‎docker/datahub-frontend/Dockerfile

+6-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ RUN apk --no-cache --update-cache --available upgrade \
2020
&& apk --no-cache add curl sqlite libc6-compat snappy \
2121
&& apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \
2222
&& apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \
23-
&& wget ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar -O opentelemetry-javaagent.jar \
23+
&& wget ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.11.0/opentelemetry-javaagent.jar -O opentelemetry-javaagent.jar \
2424
&& wget ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar
2525

2626
ENV LD_LIBRARY_PATH="/lib:/lib64"
@@ -46,6 +46,11 @@ FROM ${APP_ENV}-install AS final
4646
COPY --chown=datahub:datahub --chmod=755 ./docker/datahub-frontend/start.sh /
4747
USER datahub
4848

49+
ENV OTEL_EXPORTER_OTLP_MAX_PAYLOAD_SIZE=4194304 \
50+
OTEL_EXPORTER_OTLP_HTTP_HTTP2_MAX_FRAME_SIZE=16777215 \
51+
OTEL_EXPORTER_OTLP_HTTP_COMPRESSION=gzip \
52+
OTEL_EXPORTER_OTLP_TRACES_COMPRESSION=gzip
53+
4954
ARG SERVER_PORT=9002
5055
ENV SERVER_PORT=$SERVER_PORT
5156
RUN echo $SERVER_PORT

‎docker/datahub-gms/Dockerfile

+5-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ RUN apk --no-cache --update-cache --available upgrade \
4141
&& apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat snappy \
4242
&& apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \
4343
&& apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \
44-
&& wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
44+
&& wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.11.0/opentelemetry-javaagent.jar \
4545
&& wget --no-verbose ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
4646
&& cp /usr/lib/jvm/java-17-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
4747
COPY --from=binary /go/bin/dockerize /usr/local/bin
@@ -69,6 +69,10 @@ USER datahub
6969

7070
ENV JMX_OPTS=""
7171
ENV JAVA_OPTS=""
72+
ENV OTEL_EXPORTER_OTLP_MAX_PAYLOAD_SIZE=4194304 \
73+
OTEL_EXPORTER_OTLP_HTTP_HTTP2_MAX_FRAME_SIZE=8388608 \
74+
OTEL_EXPORTER_OTLP_HTTP_COMPRESSION=gzip \
75+
OTEL_EXPORTER_OTLP_TRACES_COMPRESSION=gzip
7276

7377
EXPOSE 8080
7478

‎docker/datahub-mae-consumer/Dockerfile

+5-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ RUN apk --no-cache --update-cache --available upgrade \
4040
&& apk --no-cache add curl bash coreutils sqlite libc6-compat snappy \
4141
&& apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \
4242
&& apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \
43-
&& wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
43+
&& wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.11.0/opentelemetry-javaagent.jar \
4444
&& wget --no-verbose ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
4545
&& cp /usr/lib/jvm/java-17-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
4646
COPY --from=binary /go/bin/dockerize /usr/local/bin
@@ -65,6 +65,10 @@ USER datahub
6565

6666
ENV JMX_OPTS=""
6767
ENV JAVA_OPTS=""
68+
ENV OTEL_EXPORTER_OTLP_MAX_PAYLOAD_SIZE=4194304 \
69+
OTEL_EXPORTER_OTLP_HTTP_HTTP2_MAX_FRAME_SIZE=16777215 \
70+
OTEL_EXPORTER_OTLP_HTTP_COMPRESSION=gzip \
71+
OTEL_EXPORTER_OTLP_TRACES_COMPRESSION=gzip
6872

6973
EXPOSE 9090
7074

‎docker/datahub-mce-consumer/Dockerfile

+5-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ RUN apk --no-cache --update-cache --available upgrade \
4040
&& apk --no-cache add curl bash sqlite libc6-compat snappy \
4141
&& apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \
4242
&& apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \
43-
&& wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
43+
&& wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.11.0/opentelemetry-javaagent.jar \
4444
&& wget --no-verbose ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
4545
&& cp /usr/lib/jvm/java-17-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
4646
COPY --from=binary /go/bin/dockerize /usr/local/bin
@@ -66,6 +66,10 @@ USER datahub
6666

6767
ENV JMX_OPTS=""
6868
ENV JAVA_OPTS=""
69+
ENV OTEL_EXPORTER_OTLP_MAX_PAYLOAD_SIZE=4194304 \
70+
OTEL_EXPORTER_OTLP_HTTP_HTTP2_MAX_FRAME_SIZE=16777215 \
71+
OTEL_EXPORTER_OTLP_HTTP_COMPRESSION=gzip \
72+
OTEL_EXPORTER_OTLP_TRACES_COMPRESSION=gzip
6973

7074
EXPOSE 9090
7175

‎docker/datahub-upgrade/Dockerfile

+13-3
Original file line numberDiff line numberDiff line change
@@ -40,15 +40,20 @@ ENV JMX_VERSION=0.20.0
4040
RUN apk --no-cache --update-cache --available upgrade \
4141
&& apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat snappy \
4242
&& apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \
43-
&& wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
44-
&& wget --no-verbose ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
43+
&& mkdir -p /datahub/datahub-upgrade/lib \
44+
&& wget --no-verbose -P /datahub/datahub-upgrade/lib ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.11.0/opentelemetry-javaagent.jar \
45+
&& wget --no-verbose -P /datahub/datahub-upgrade/lib ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
46+
&& chmod -R a+r /datahub/datahub-upgrade/lib \
4547
&& cp /usr/lib/jvm/java-17-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
4648
COPY --from=binary /go/bin/dockerize /usr/local/bin
4749

4850
ENV LD_LIBRARY_PATH="/lib:/lib64"
4951

5052
FROM base AS prod-install
5153
COPY datahub-upgrade.jar /datahub/datahub-upgrade/bin/
54+
COPY docker/datahub-upgrade/start.sh /datahub/datahub-upgrade/scripts/start.sh
55+
COPY docker/monitoring/client-prometheus-config.yaml /datahub/datahub-upgrade/scripts/prometheus-config.yaml
56+
RUN chmod +x /datahub/datahub-upgrade/scripts/start.sh
5257
COPY metadata-models/src/main/resources/entity-registry.yml /datahub/datahub-gms/resources/entity-registry.yml
5358

5459
FROM base AS dev-install
@@ -60,4 +65,9 @@ FROM ${APP_ENV}-install AS final
6065
RUN addgroup -S datahub && adduser -S datahub -G datahub
6166
USER datahub
6267

63-
ENTRYPOINT ["java", "-jar", "/datahub/datahub-upgrade/bin/datahub-upgrade.jar"]
68+
ENV OTEL_EXPORTER_OTLP_MAX_PAYLOAD_SIZE=4194304 \
69+
OTEL_EXPORTER_OTLP_HTTP_HTTP2_MAX_FRAME_SIZE=16777215 \
70+
OTEL_EXPORTER_OTLP_HTTP_COMPRESSION=gzip \
71+
OTEL_EXPORTER_OTLP_TRACES_COMPRESSION=gzip
72+
73+
ENTRYPOINT ["/datahub/datahub-upgrade/scripts/start.sh"]

‎docker/datahub-upgrade/start.sh

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/bin/bash
2+
3+
# Add default URI (http) scheme if needed
4+
if ! echo $NEO4J_HOST | grep -q "://" ; then
5+
NEO4J_HOST="http://$NEO4J_HOST"
6+
fi
7+
8+
if [[ ! -z $ELASTICSEARCH_USERNAME ]] && [[ -z $ELASTICSEARCH_AUTH_HEADER ]]; then
9+
AUTH_TOKEN=$(echo -ne "$ELASTICSEARCH_USERNAME:$ELASTICSEARCH_PASSWORD" | base64 --wrap 0)
10+
ELASTICSEARCH_AUTH_HEADER="Authorization:Basic $AUTH_TOKEN"
11+
fi
12+
13+
# Add default header if needed
14+
if [[ -z $ELASTICSEARCH_AUTH_HEADER ]]; then
15+
ELASTICSEARCH_AUTH_HEADER="Accept: */*"
16+
fi
17+
18+
if [[ $ELASTICSEARCH_USE_SSL == true ]]; then
19+
ELASTICSEARCH_PROTOCOL=https
20+
else
21+
ELASTICSEARCH_PROTOCOL=http
22+
fi
23+
24+
WAIT_FOR_EBEAN=""
25+
if [[ $SKIP_EBEAN_CHECK != true ]]; then
26+
if [[ $ENTITY_SERVICE_IMPL == ebean ]] || [[ -z $ENTITY_SERVICE_IMPL ]]; then
27+
WAIT_FOR_EBEAN=" -wait tcp://$EBEAN_DATASOURCE_HOST "
28+
fi
29+
fi
30+
31+
WAIT_FOR_CASSANDRA=""
32+
if [[ $ENTITY_SERVICE_IMPL == cassandra ]] && [[ $SKIP_CASSANDRA_CHECK != true ]]; then
33+
WAIT_FOR_CASSANDRA=" -wait tcp://$CASSANDRA_DATASOURCE_HOST "
34+
fi
35+
36+
WAIT_FOR_KAFKA=""
37+
if [[ $SKIP_KAFKA_CHECK != true ]]; then
38+
WAIT_FOR_KAFKA=" -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') "
39+
fi
40+
41+
WAIT_FOR_NEO4J=""
42+
if [[ $GRAPH_SERVICE_IMPL != elasticsearch ]] && [[ $SKIP_NEO4J_CHECK != true ]]; then
43+
WAIT_FOR_NEO4J=" -wait $NEO4J_HOST "
44+
fi
45+
46+
OTEL_AGENT=""
47+
if [[ $ENABLE_OTEL == true ]]; then
48+
OTEL_AGENT="-javaagent:/datahub/datahub-upgrade/lib/opentelemetry-javaagent.jar "
49+
if [ -z "${OTEL_EXPORTER_OTLP_MAX_PAYLOAD_SIZE}" ]; then
50+
export OTEL_EXPORTER_OTLP_MAX_PAYLOAD_SIZE=4194304
51+
fi
52+
if [ -z "${OTEL_EXPORTER_OTLP_HTTP_HTTP2_MAX_FRAME_SIZE}" ]; then
53+
export OTEL_EXPORTER_OTLP_HTTP_HTTP2_MAX_FRAME_SIZE=16777215
54+
fi
55+
fi
56+
57+
PROMETHEUS_AGENT=""
58+
if [[ $ENABLE_PROMETHEUS == true ]]; then
59+
PROMETHEUS_AGENT="-javaagent:/datahub/datahub-upgrade/lib/jmx_prometheus_javaagent.jar=4318:/datahub/datahub-gms/scripts/prometheus-config.yaml "
60+
fi
61+
62+
COMMON="
63+
$WAIT_FOR_EBEAN \
64+
$WAIT_FOR_CASSANDRA \
65+
$WAIT_FOR_KAFKA \
66+
$WAIT_FOR_NEO4J \
67+
-timeout 240s \
68+
java $JAVA_OPTS $JMX_OPTS \
69+
$OTEL_AGENT \
70+
-Dstats=unsecure \
71+
-Dloader.path=/datahub/datahub-upgrade/lib \
72+
-jar /datahub/datahub-upgrade/bin/datahub-upgrade.jar"
73+
74+
if [[ $SKIP_ELASTICSEARCH_CHECK != true ]]; then
75+
exec /usr/local/bin/dockerize \
76+
-wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT -wait-http-header "$ELASTICSEARCH_AUTH_HEADER" \
77+
$COMMON "$@"
78+
else
79+
exec /usr/local/bin/dockerize $COMMON "$@"
80+
fi

‎docs/advanced/monitoring.md

+5
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,11 @@ the `@WithSpan` annotation in various places to make the trace more readable. Yo
3333
tracing collector of choice. Our example [docker-compose](../../docker/monitoring/docker-compose.monitoring.yml) deploys
3434
an instance of Jaeger with port 16686. The traces should be available at http://localhost:16686.
3535

36+
### Configuration Note
37+
38+
We recommend using either `grpc` or `http/protobuf`, configured using `OTEL_EXPORTER_OTLP_PROTOCOL`. Avoid using `http` will not work as expected due to the size of
39+
the generated spans.
40+
3641
## Metrics
3742

3843
With tracing, we can observe how a request flows through our system into the persistence layer. However, for a more

‎metadata-jobs/mae-consumer-job/build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ dependencies {
2929
implementation externalDependency.log4j2Api
3030
compileOnly externalDependency.lombok
3131
implementation externalDependency.logbackClassic
32+
runtimeOnly externalDependency.opentelemetryExporter
3233

3334
testImplementation project(':metadata-dao-impl:kafka-producer')
3435
testImplementation project(':metadata-jobs:pe-consumer')

‎metadata-jobs/mce-consumer-job/build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ dependencies {
3131
implementation externalDependency.log4j2Api
3232
compileOnly externalDependency.lombok
3333
implementation externalDependency.logbackClassic
34+
runtimeOnly externalDependency.opentelemetryExporter
3435

3536
runtimeOnly externalDependency.mariadbConnector
3637
runtimeOnly externalDependency.mysqlConnector

‎metadata-service/war/build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ dependencies {
4343
implementation externalDependency.springWebMVC
4444
implementation externalDependency.springBootAutoconfigure
4545
implementation externalDependency.servletApi
46+
runtimeOnly externalDependency.opentelemetryExporter
4647

4748
implementation spec.product.pegasus.restliDocgen
4849
implementation spec.product.pegasus.restliSpringBridge

0 commit comments

Comments
 (0)