Remove telegraf and use only fluent-bit for telemetry (#1030)

[comment]: # (Note that your PR title should follow the conventional
commit format: https://conventionalcommits.org/en/v1.0.0/#summary)
# PR Description

- Upgrade `fluent-bit`
  - Linux: `2.1.10` -> `3.2.2` (latest)
- `>=3.2` is necessary for using `metrics_selector` and `labels`
processors for filtering Prometheus metrics
  - Windows: `2.1.10` -> `3.0.7` (latest)
     - `3.0` is the latest Windows version
- Remove `telegraf` for Linux and Windows
- Changes to `fluent-bit`
   - Built-in Plugins:
- Use `prometheus_scrape` input plugin to scrape Prometheus metrics
previously collected by telegraf
- Use `metrics_selector` and `labels` processors to filter certain
metrics and drop unnecessary labels before sending to App Insights
   - Conf:
- Use new YAML format for config to be able to use `metrics_selector`
and `labels` processors
   - Custom Output Plugin:
1. Collect CPU and Memory usage for otelcollector and metricsextension
that were previously collected by telegraf
- This runs as a go routine, separate from the fluent-bit pipeline
- Use the same underlying golang package as telegraf:
`github.com/shirou/gopsutil/v4/process`
- Collect at the same frequency as telegraf and aggregate to p50 and p95
          - Send extra env var as customDimensions as telegraf did 
3. Decode the Prometheus metrics msgpack from fluent-bit and send to App
Insights in the format we want
- Add one line to the fluent-bit `proxy_plugin` file so that the
Prometheus metrics will be allowed to flow to our golang output plugin:
      - `out->event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS;`
- `fluent-bit` has the `proxy_plugin` files to allow the golang output
plugins to be built upon the C code. However, this does not specify what
type the output plugin accepts (out of `logs`, `metrics`, and `traces`
types), so it defaults to only allowing the `logs` type to be routed to
the ouput plugin.
- Build Pipeline:
- Build `fluent-bit` with the line added above in the exact same way
Mariner builds the package.
- Only build `fluent-bit` with the plugins that we actually use so that
our CVE surface area is very low.
- Main image bug fixes:
- Use daemonset config file for fluent-bit for the daemonset.
Previously, it was using the replicaset config file for both replicaset
and daemonset and the daemonset logs weren't being collected
- Fix telemetry sent for `network-observability` and `acstor` that was
missed

[comment]: # (The below checklist is for PRs adding new features. If a
box is not checked, add a reason why it's not needed.)
# New Feature Checklist

- [x] Link to the one-pager about the feature:
https://msazure.visualstudio.com/InfrastructureInsights/_wiki/wikis/InfrastructureInsights.wiki/741581/TelegrafRemoval
- [x] Attach results of scale and perf testing:
<img width="1753" alt="image"
src="https://github.com/user-attachments/assets/de8a4d9a-6f0f-40ba-b55d-491dc68e503b"
/>

# Telemetry Values Comparison

- ReplicaSet
<img width="1712" alt="image"
src="https://github.com/user-attachments/assets/4ab3d9b3-51ab-4775-a5a6-872d3d0291c9"
/>

- DaemonSet
<img width="1706" alt="image"
src="https://github.com/user-attachments/assets/cd2feee3-1bfd-411c-9050-99a3681f00c8"
/>

- All extra [env var telemetry are transferred
over](https://dataexplorer.azure.com/dashboards/94da59c1-df12-4134-96bb-82c6b32e6199?p-_cluster=v-%2Fsubscriptions%2Fb9842c7c-1a38-4385-8f39-a51314758bcf%2FresourceGroups%2Fgrace-win%2Fproviders%2FMicrosoft.ContainerService%2FmanagedClusters%2Fgrace-win&p-_startTime=7days&p-_endTime=now&p-Interval=v-5m&p-AKSClusterID=v-675b8ceeceb9d100010e6fe2#9d5aa5eb-cc6f-46c9-81f6-22c8fe5357e4)

Author: gracewehner

.pipelines/azure-pipeline-build.yml

@@ -1,45 +1,25 @@
 # CRITICAL
-# telegraf
-CVE-2024-41110
 # kube-state-metrics
-CVE-2024-24790
+CVE-2024-45337 # golang.org/x/crypto
+CVE-2024-24790 # stdlib
 # HIGH
-# telegraf
-GHSA-7jwh-3vrq-q3m8
-CVE-2024-27289
-CVE-2024-27304
+# otelcollector
+CVE-2024-45338 # golang.org/x/net
+# promconfigvalidator
+CVE-2024-45338 # golang.org/x/net
+# configurationreader
+CVE-2024-45338 # golang.org/x/net
+# targetallocator
+CVE-2024-45338 # golang.org/x/net
 # kube-state-metrics
-CVE-2024-34156
-# node-exporter
-CVE-2023-39325
-CVE-2023-29403
-CVE-2023-39325
-CVE-2023-45283
+CVE-2024-45338 # golang.org/x/net
+CVE-2024-34156 # stdlib
 # MEDIUM
-# telegraf
-CVE-2024-35255
-CVE-2024-28110
-CVE-2024-24557
-CVE-2024-29018
-CVE-2023-45288
-CVE-2024-24786
 # kube-state-metrics
-CVE-2024-24789
-CVE-2024-24791
-CVE-2024-34155
-CVE-2024-34158
-# node-exporter
-CVE-2023-48795
-CVE-2023-3978
-CVE-2023-44487
-CVE-2023-29406
-CVE-2023-29409
-CVE-2023-39318
-CVE-2023-39319
-CVE-2023-39326
-CVE-2023-45284
-CVE-2023-45289 
-CVE-2023-45290
-CVE-2024-24783
-CVE-2024-24784
-CVE-2024-24785
+CVE-2023-45288 # stdlib
+CVE-2024-24789 # stdlib
+CVE-2024-24791 # stdlib
+CVE-2024-34155 # stdlib
+CVE-2024-34158 # stdlib
+CVE-2024-45336 # stdlib
+CVE-2024-45341 # stdlib

.trivyignore

@@ -1,5 +1,6 @@
 
-FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 as builder
+ARG GOLANG_VERSION
+FROM mcr.microsoft.com/oss/go/microsoft/golang:${GOLANG_VERSION} as builder
 
 # Set necessary environmet variables needed for our image
 ENV GO111MODULE=on \
@@ -10,12 +11,10 @@ ENV GO111MODULE=on \
 # Move to working directory /build
 WORKDIR /build
 
-ARG GOLANG_VERSION
-
 # Copy and download dependency using go mod
 COPY go.mod .
 COPY go.sum .
-RUN tdnf install -y golang-${GOLANG_VERSION} ca-certificates
+#RUN tdnf install -y golang-${GOLANG_VERSION} ca-certificates
 RUN go mod download
 
 # COPY client-cert.pem /etc/prometheus/certs/

internal/referenceapp/golang/linux/Dockerfile

@@ -68,6 +68,39 @@ RUN apt-get update && apt-get install gcc-aarch64-linux-gnu -y
 ARG TARGETOS TARGETARCH
 RUN if [ "$TARGETARCH" = "arm64" ] ; then CC=aarch64-linux-gnu-gcc CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -buildmode=exe -ldflags '-linkmode external -extldflags=-Wl,-z,now' -o main.exe ./main.go ; else CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -buildmode=exe -ldflags '-linkmode external -extldflags=-Wl,-z,now' -o main.exe ./main.go ; fi
 
+ARG TARGETARCH
+FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 AS fluent-bit-binary-builder
+WORKDIR /
+# Install with the same exact dependencies and code source that Mariner uses
+RUN tdnf install wget tar ca-certificates bison cmake cyrus-sasl-devel doxygen flex gcc-c++ \
+    gnutls-devel graphviz libpq-devel libyaml-devel luajit-devel make openssl-devel pkgconfig \
+    systemd-devel systemd-rpm-macros zlib-devel build-essential -y
+ARG FLUENT_BIT_VERSION
+RUN wget https://github.com/fluent/fluent-bit/archive/refs/tags/v${FLUENT_BIT_VERSION}.tar.gz
+RUN tar -xvf v${FLUENT_BIT_VERSION}.tar.gz
+# Add a file with settings to build only the plugins we use
+COPY ./fluent-bit/plugins_options.cmake /fluent-bit-${FLUENT_BIT_VERSION}/cmake/plugins_options.cmake
+# Make a change that allows Fluent-Bit metrics to flow to our Go output plugin
+RUN sed -i '/out->type  = FLB_OUTPUT_PLUGIN_PROXY;/a \ \ \ \ out->event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS;' /fluent-bit-${FLUENT_BIT_VERSION}/src/flb_plugin_proxy.c
+WORKDIR /fluent-bit-${FLUENT_BIT_VERSION}/build
+# Run cmake with the same flags that Mariner uses
+RUN cmake \
+-DCMAKE_BUILD_TYPE=RelWithDebInfo \
+-DFLB_EXAMPLES=Off \
+-DFLB_OUT_SLACK=Off \
+-DFLB_IN_SYSTEMD=On \
+-DFLB_OUT_TD=Off \
+-DFLB_OUT_ES=Off \
+-DFLB_SHARED_LIB=On \
+-DFLB_RELEASE=On \
+-DFLB_DEBUG=Off \
+-DFLB_TLS=On \
+-DFLB_JEMALLOC=On \
+-DFLB_PREFER_SYSTEM_LIBS=On \
+-DFLB_PROXY_GO=On ../
+RUN make
+RUN make install
+
 FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 as builder
 LABEL description="Azure Monitor Prometheus metrics collector"
 LABEL maintainer="[email protected]"
@@ -103,9 +136,11 @@ COPY --from=main-builder --chmod=777 /main/main.exe $tmpdir/main
 
 COPY ./scripts/*.sh $tmpdir/
 COPY ./metricextension/me.config ./metricextension/me_internal.config ./metricextension/me_ds.config ./metricextension/me_ds_internal.config /usr/sbin/
-COPY ./telegraf/ $tmpdir/telegraf/
-COPY ./fluent-bit/fluent-bit.conf ./fluent-bit/fluent-bit-daemonset.conf ./fluent-bit/fluent-bit-parsers.conf $tmpdir/fluent-bit/
+COPY ./fluent-bit/fluent-bit.yaml ./fluent-bit/fluent-bit-daemonset.yaml ./fluent-bit/fluent-bit-parsers.conf $tmpdir/fluent-bit/
 COPY --from=fluent-bit-builder /src/out_appinsights.so $tmpdir/fluent-bit/bin/
+COPY --from=fluent-bit-binary-builder /usr/local/bin/fluent-bit /usr/local/bin/fluent-bit
+COPY --from=fluent-bit-binary-builder /usr/local/etc/fluent-bit /usr/local/etc/fluent-bit
+COPY --from=fluent-bit-binary-builder /usr/local/lib/fluent-bit /usr/local/etc/fluent-bit
 COPY ./react /static/react
 COPY ./LICENSE $tmpdir/microsoft
 COPY ./NOTICE $tmpdir/microsoft
@@ -123,7 +158,7 @@ RUN chmod 775 $tmpdir/*.sh;
 RUN sync;
 RUN $tmpdir/setup.sh ${TARGETARCH}
 # If wanting to run without distroless, uncomment this line and comment everything after
-# CMD [ "/opt/main.sh" ]
+# ENTRYPOINT ["./opt/main"]
 
 FROM mcr.microsoft.com/cbl-mariner/distroless/base:2.0
 # Below is for ContainerInsightsPrometheusCollector-Prod AppInsights Resource
@@ -169,8 +204,6 @@ COPY --from=builder /usr/sbin/MetricsExtension /usr/sbin/MetricsExtension
 COPY --from=builder /usr/bin/inotifywait /usr/bin/inotifywait
 COPY --from=builder /usr/bin/bash /usr/bin/bash
 COPY --from=builder /usr/sbin/busybox /usr/sbin/busybox
-COPY --from=builder /usr/bin/fluent-bit /usr/bin/fluent-bit
-COPY --from=builder /usr/bin/telegraf /usr/bin/telegraf
 COPY --from=builder /usr/sbin/crond /usr/sbin/crond
 COPY --from=builder /usr/bin/vim /usr/bin/vim
 COPY --from=builder /usr/share/vim /usr/share/vim
@@ -183,6 +216,10 @@ COPY --from=builder /bin/sh /bin/sh
 COPY --from=builder /usr/bin/p11-kit /usr/bin
 COPY --from=builder /usr/bin/trust /usr/bin
 
+COPY --from=fluent-bit-binary-builder /usr/local/bin/fluent-bit /usr/local/bin/fluent-bit
+COPY --from=fluent-bit-binary-builder /usr/local/etc/fluent-bit /usr/local/etc/fluent-bit
+COPY --from=fluent-bit-binary-builder /usr/local/lib/fluent-bit /usr/local/etc/fluent-bit
+
 # bash dependencies
 COPY --from=builder /lib/libreadline.so.8 /lib/
 COPY --from=builder /usr/lib/libncursesw.so.6 /usr/lib/libtinfo.so.6 /usr/lib/
@@ -198,9 +235,7 @@ COPY --from=builder /lib/libboost_filesystem.so.1.76.0 /lib/libcpprest.so.2.10
 COPY --from=builder /lib64/libuuid.so.1 /lib64
 # fluent-bit dependencies
 # libssl.so.1.1 & libcrypto.so.1.1 are already available with openssl in distroless and copying them over causes FIPS HMAC verification failures
-COPY --from=builder /lib/libyaml-0.so.2 /lib/libsystemd.so.0 /lib/libcurl.so.4 /lib/libm.so.6 /lib/libz.so.1 /lib/libzstd.so.1 /lib/libsasl2.so.3 /lib/libgcc_s.so.1 /lib/libc.so.6 /lib/liblzma.so.5 /lib/liblz4.so.1  /lib/libcap.so.2 /lib/libgcrypt.so.20 /lib/libnghttp2.so.14 /lib/libssh2.so.1 /lib/libgssapi_krb5.so.2 /lib/libresolv.so.2 /lib/libgpg-error.so.0 /usr/lib/libkrb5.so.3 /usr/lib/libk5crypto.so.3 /usr/lib/libcom_err.so.2 /usr/lib/libkrb5support.so.0 /lib/
-# telegraf dependencies
-COPY --from=builder /lib/libc.so.6 /lib/
+COPY --from=fluent-bit-binary-builder /lib/libluajit-5.1.so.2 /lib/libyaml-0.so.2 /lib/libsystemd.so.0 /lib/libgcc_s.so.1 /lib/libc.so.6 /lib/liblzma.so.5 /lib/libzstd.so.1 /lib/liblz4.so.1 /lib/libcap.so.2 /lib/libgcrypt.so.20 /lib/libgpg-error.so.0 /lib/
 # mdsd dependencies
 COPY --from=builder /usr/lib/libdl.so.2 /usr/lib/librt.so.1 /usr/lib/libpthread.so.0 /usr/lib/libm.so.6 /usr/lib/libstdc++.so.6 /usr/lib/libgcc_s.so.1 /usr/lib/
 # logrotate dependencies

otelcollector/build/linux/Dockerfile

@@ -30,7 +30,6 @@ COPY ./configmapparser/default-prom-configs/*.yml $tmpdir/microsoft/otelcollecto
 COPY ./opentelemetry-collector-builder/otelcollector.exe ./opentelemetry-collector-builder/collector-config-default.yml ./opentelemetry-collector-builder/collector-config-template.yml $tmpdir/microsoft/otelcollector/
 COPY ./prom-config-validator-builder/promconfigvalidator.exe $tmpdir/
 COPY ./metricextension/me.config ./metricextension/me_internal.config ./metricextension/me_ds.config ./metricextension/me_ds_win.config ./metricextension/me_ds_internal.config ./metricextension/me_ds_internal_win.config $tmpdir/metricextension/
-COPY ./telegraf/telegraf-prometheus-collector-windows.conf $tmpdir/telegraf/
 COPY ./fluent-bit/fluent-bit-windows.conf $tmpdir/fluent-bit/
 COPY ./fluent-bit/fluent-bit-parsers.conf $tmpdir/fluent-bit/
 COPY ./fluent-bit/src/out_appinsights.so $tmpdir/fluent-bit/bin/

otelcollector/build/windows/Dockerfile

@@ -32,7 +32,7 @@ Write-Host ('Installing Fluent Bit');
 try {
     # Keep version in sync with linux in setup.sh file
     # $fluentBitUri = 'https://github.com/microsoft/OMS-docker/releases/download/winakslogagent/td-agent-bit-1.4.0-win64.zip'
-    $fluentBitUri = 'https://releases.fluentbit.io/2.1/fluent-bit-2.1.10-win64.zip'
+    $fluentBitUri = 'https://releases.fluentbit.io/3.0/fluent-bit-3.0.7-win64.zip'
     Invoke-WebRequest -Uri $fluentBitUri -OutFile /installation/fluent-bit.zip
     Expand-Archive -Path /installation/fluent-bit.zip -Destination /installation/fluent-bit
     Move-Item -Path /installation/fluent-bit/*/bin/* -Destination /opt/fluent-bit/bin/ -ErrorAction SilentlyContinue