Test: query for $ and $$ label for CRs and external labels (#1050)

[comment]: # (Note that your PR title should follow the conventional
commit format: https://conventionalcommits.org/en/v1.0.0/#summary)
# PR Description
Add to the query tests to check for $ and $$ labels on PodMonitor and
ServiceMonitor and to check for external labels

Author: gracewehner

.pipelines/azure-pipeline-build.yml

@@ -84,6 +84,10 @@ stages:
           WINDOWS_2019_BASE_IMAGE_VERSION=ltsc2019
           WINDOWS_2022_BASE_IMAGE_VERSION=ltsc2022
 
+          #Truncating this to 123 characters to add the conf prefix
+          ARC_CONFORMANCE_TAG_SUFFIX=$(echo "${LINUX_IMAGE_TAG}" | cut -c1-123)
+          ARC_CONFORMANCE_IMAGE_TAG=conf-$ARC_CONFORMANCE_TAG_SUFFIX
+
           LINUX_FULL_IMAGE_NAME=$ACR_REGISTRY$ACR_REPOSITORY:$LINUX_IMAGE_TAG
           TARGET_ALLOCATOR_FULL_IMAGE_NAME=$ACR_REGISTRY$ACR_REPOSITORY:$TARGET_ALLOCATOR_IMAGE_TAG
           LINUX_CONFIG_READER_FULL_IMAGE_NAME=$ACR_REGISTRY$ACR_REPOSITORY:$LINUX_CONFIG_READER_IMAGE_TAG
@@ -95,6 +99,7 @@ stages:
           LINUX_REF_APP_PYTHON_FULL_IMAGE_NAME=$ACR_REGISTRY$ACR_REPOSITORY:$LINUX_REF_APP_PYTHON_IMAGE_TAG
           WINDOWS_REF_APP_GOLANG_FULL_IMAGE_NAME=$ACR_REGISTRY$ACR_REPOSITORY:$WIN_REF_APP_GOLANG_IMAGE_TAG
           WINDOWS_REF_APP_PYTHON_FULL_IMAGE_NAME=$ACR_REGISTRY$ACR_REPOSITORY:$WIN_REF_APP_PYTHON_IMAGE_TAG
+          ARC_CONFORMANCE_FULL_IMAGE_NAME=$ACR_REGISTRY$ACR_REPOSITORY:$ARC_CONFORMANCE_IMAGE_TAG
 
           echo "##vso[build.updatebuildnumber]$SEMVER"
           echo "##vso[task.setvariable variable=SEMVER;isOutput=true]$SEMVER"
@@ -116,6 +121,8 @@ stages:
           echo "##vso[task.setvariable variable=ARC_HELM_CHART_NAME;isOutput=true]$ARC_HELM_CHART_NAME"
           echo "##vso[task.setvariable variable=HELM_FULL_IMAGE_NAME;isOutput=true]$HELM_FULL_IMAGE_NAME"
           echo "##vso[task.setvariable variable=ARC_HELM_FULL_IMAGE_NAME;isOutput=true]$ARC_HELM_FULL_IMAGE_NAME"
+          echo "##vso[task.setvariable variable=ARC_CONFORMANCE_FULL_IMAGE_NAME;isOutput=true]$ARC_CONFORMANCE_FULL_IMAGE_NAME"
+
         displayName: 'Build: set image registry, repo, and tags'
         name: setup
 
@@ -510,6 +517,31 @@ stages:
         displayName: "Download, install Oras and run oras attach"
         condition: succeeded()        
 
+  - job: Arc_Conformace_Test_Image
+    displayName: "Build: Arc Conformance Test Image"
+    pool:
+      name: Azure-Pipelines-CI-Test-EO
+    dependsOn: Image_Tags_and_Ev2_Artifacts
+    variables:
+      skipComponentGovernanceDetection: true
+      ARC_CONFORMANCE_FULL_IMAGE_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.ARC_CONFORMANCE_FULL_IMAGE_NAME'] ]
+    # This is necessary because of: https://github.com/moby/moby/issues/37965
+      DOCKER_BUILDKIT: 1
+    condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)))
+    steps:
+      - checkout: self
+        persistCredentials: true
+      - bash: |
+          docker pull mcr.microsoft.com/azuremonitor/containerinsights/cidev/prometheus-collector/images:buildx-stable-1
+          
+          docker buildx create --name dockerbuilder --driver docker-container --driver-opt image=mcr.microsoft.com/azuremonitor/containerinsights/cidev/prometheus-collector/images:buildx-stable-1 --use 
+          docker buildx inspect --bootstrap
+          docker login containerinsightsprod.azurecr.io -u $(ACR_USERNAME) -p $(ACR_PASSWORD)
+          docker buildx build . -f arc-conformance/Dockerfile -t $(ARC_CONFORMANCE_FULL_IMAGE_NAME) --push
+          docker pull $(ARC_CONFORMANCE_FULL_IMAGE_NAME)  
+        workingDirectory: $(Build.SourcesDirectory)/otelcollector/test/
+        displayName: "Build: build and push Arc conformance test image to dev ACR"
+
   - job: SDL_Policheck_Scan
     displayName: "SDL: policheck scanning"
     pool:
@@ -1940,7 +1972,7 @@ stages:
               kubectl apply -f ./testkube/testkube-test-crs-ci-dev-arc-wcus.yaml
               kubectl apply -f ./test-cluster-yamls/configmaps/ama-metrics-prometheus-config-configmap.yaml
               kubectl apply -f ./test-cluster-yamls/configmaps/ama-metrics-prometheus-config-node-configmap.yaml
-              kubectl apply -f ./test-cluster-yamls/configmaps/ama-metrics-prometheus-config-node-windows=configmap.yaml
+              kubectl apply -f ./test-cluster-yamls/configmaps/ama-metrics-prometheus-config-node-windows-configmap.yaml
               kubectl apply -f ./test-cluster-yamls/configmaps/ama-metrics-settings-configmap.yaml
               kubectl apply -f ./test-cluster-yamls/customresources/prometheus-reference-app.yaml
 
@@ -2191,7 +2223,7 @@ stages:
               kubectl apply -f ./testkube/testkube-test-crs-ci-dev-aks-mac-eus.yaml
               kubectl apply -f ./test-cluster-yamls/configmaps/ama-metrics-prometheus-config-configmap.yaml
               kubectl apply -f ./test-cluster-yamls/configmaps/ama-metrics-prometheus-config-node-configmap.yaml
-              kubectl apply -f ./test-cluster-yamls/configmaps/ama-metrics-prometheus-config-node-windows=configmap.yaml
+              kubectl apply -f ./test-cluster-yamls/configmaps/ama-metrics-prometheus-config-node-windows-configmap.yaml
               kubectl apply -f ./test-cluster-yamls/configmaps/ama-metrics-settings-configmap.yaml
               kubectl apply -f ./test-cluster-yamls/customresources/prometheus-reference-app.yaml
 

otelcollector/test/ginkgo-e2e/querymetrics/query_metrics_test.go

@@ -248,35 +248,58 @@ var _ = Describe("Query Metrics Test Suite", func() {
 	)
 
 	DescribeTable("should return the expected labels for specified metrics in each job",
-		func(job string, metric string, labels []string) {
-			for _, label := range labels {
-				query := fmt.Sprintf("%s{job=\"%s\"}", metric, job)
+		func(job string, metric string, labels map[string]string) {
+			query := fmt.Sprintf("%s{job=\"%s\"}", metric, job)
 
-				warnings, result, err := utils.InstantQuery(PrometheusQueryClient, query)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(warnings).To(BeEmpty())
+			warnings, result, err := utils.InstantQuery(PrometheusQueryClient, query)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(warnings).To(BeEmpty())
 
-				vectorResult, ok := result.(model.Vector)
-				Expect(ok).To(BeTrue(), "result should be of type model.Vector for metric %s", metric)
-				Expect(vectorResult).NotTo(BeEmpty(), "Metric %s is missing", metric)
+			vectorResult, ok := result.(model.Vector)
+			Expect(ok).To(BeTrue(), "result should be of type model.Vector for metric %s", metric)
+			Expect(vectorResult).NotTo(BeEmpty(), "Metric %s is missing", metric)
 
-				for _, sample := range vectorResult {
+			for _, sample := range vectorResult {
+				for label, expectedValue := range labels {
 					val, ok := sample.Metric[model.LabelName(label)]
-					Expect(ok).To(BeTrue(), fmt.Sprintf("Expected label %q not found in metric %q", label, metric))
-					Expect(val).NotTo(BeEmpty(), fmt.Sprintf("Label %q is empty in metric %q", label, metric))
-					Expect(sample.Value.String()).To(Equal("1"))
+					Expect(ok).To(BeTrue(), fmt.Sprintf("Expected label %q not found in metric %q for the job %s", label, metric, job))
+					Expect(string(val)).To(MatchRegexp(expectedValue), fmt.Sprintf("Label %q in metric %q for job %s has unexpected value: %s", label, metric, job, val))
 				}
 			}
 		},
-		Entry("Metric relabeling with dollar signs", "prometheus_ref_app", "up", []string{
-			"double_dollar_sign",
-			"single_dollar_sign",
+		Entry("Relabeling with dollar signs", "prometheus_ref_app", "up", map[string]string{
+			"double_dollar_sign": "prometheus-reference-app", // Legacy backwards compatibility for $$1 when single $ was not supported
+			"single_dollar_sign": "prometheus-reference-app",
+		}),
+		Entry("Relabeling with $NODE_NAME and $NODE_IP", "node-configmap", "up", map[string]string{
+			"node_name_single_dollar_sign": ".+", // Node Name and IP env var substitution is only for daemonset
+			"node_ip_single_dollar_sign":   "\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}",
+			"node_name_double_dollar_sign": ".+",                                        // Legacy backwards compatibility for $$NODE_NAME when single $ was not supported
+			"node_ip_double_dollar_sign":   "\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}", // Legacy backwards compatibility for $$NODE_IP when single $ was not supported
+		}),
+		Entry("Relabeling with dollar signs & external labels for PodMonitor", "default/referenceapp", "up", map[string]string{
+			"double_dollar_sign": "\\$1",                     // PodMonitor does not have the legacy backwards compatibility for $$1
+			"single_dollar_sign": "prometheus-reference-app", // $1 does work for PodMonitor
+			"external_label_1":   "external_label_value",
+			"external_label_123": "external_label_value",
+		}),
+		Entry("Relabeling with dollar signs & external labels for ServiceMonitor", "prometheus-reference-service", "up", map[string]string{
+			"double_dollar_sign": "\\$1",                     // ServiceMonitor does not have the legacy backwards compatibility for $$1
+			"single_dollar_sign": "prometheus-reference-app", // $1 does work for ServiceMonitor
+			"external_label_1":   "external_label_value",
+			"external_label_123": "external_label_value",
+		}),
+		Entry("External labels are applied from ReplicaSet Configmap", "prometheus_ref_app", "up", map[string]string{
+			"external_label_1":   "external_label_value",
+			"external_label_123": "external_label_value",
+		}),
+		Entry("External labels are applied from DaemonSet Configmap", "node-configmap", "up", map[string]string{
+			"external_label_1":   "external_label_value",
+			"external_label_123": "external_label_value",
 		}),
-		Entry("Metric relabeling with $NODE_NAME and $NODE_IP", "node-configmap", "up", []string{
-			"node_name_single_dollar_sign",
-			"node_ip_single_dollar_sign",
-			"node_name_double_dollar_sign",
-			"node_ip_double_dollar_sign",
+		Entry("External labels are applied from Windows DaemonSet Configmap", "windows-node-configmap", "up", map[string]string{
+			"external_label_1":   "external_label_value",
+			"external_label_123": "external_label_value",
 		}),
 	)
 

otelcollector/test/test-cluster-yamls/configmaps/ama-metrics-prometheus-config-node-windows-configmap.yaml

@@ -2,10 +2,34 @@ kind: ConfigMap
 apiVersion: v1
 data:
   prometheus-config: |-
+    global:
+      external_labels:
+        external_label_1: "external_label_value"
+        external_label_123: "external_label_value"
     scrape_configs:
-    - job_name: empty
+    - job_name: windows-node-configmap
+      scheme: https
+      metrics_path: /metrics
+      scrape_interval: 30s
+      label_limit: 63
+      label_name_length_limit: 511
+      label_value_length_limit: 1023
+      tls_config:
+        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        insecure_skip_verify: true
+      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+      relabel_configs:
+      - source_labels: [__metrics_path__]
+        regex: (.*)
+        target_label: metrics_path
+      - source_labels: [__address__]
+        replacement: '$NODE_NAME'
+        target_label: instance
+      - source_labels: [__address__]
+        replacement: '$OS_TYPE'
+        target_label: "kubernetes_io_os"
       static_configs:
-      - targets: []
+      - targets: ['$NODE_IP:10250']
 metadata:
   name: ama-metrics-prometheus-config-node-windows
   namespace: kube-system

otelcollector/test/test-cluster-yamls/customresources/prometheus-referernce-app.yaml otelcollector/test/test-cluster-yamls/customresources/prometheus-reference-app.yaml

@@ -10,17 +10,17 @@ spec:
   labelValueLengthLimit: 1023
   podMetricsEndpoints:
   - relabelings:
-    - source_labels: [__meta_kubernetes_pod_label_app]
+    - sourceLabels: [__meta_kubernetes_pod_label_app]
       action: replace
       regex: (.*)
       replacement: $$1
-      target_label: double_dollar_sign
-    - source_labels: [__meta_kubernetes_pod_label_app]
+      targetLabel: double_dollar_sign
+    - sourceLabels: [__meta_kubernetes_pod_label_app]
       action: replace
       regex: (.*)
       replacement: $1
-      target_label: single_dollar_sign
-  - metricRelabelings:
+      targetLabel: single_dollar_sign
+    metricRelabelings:
       - action: keep
         regex: up
         sourceLabels:
@@ -43,17 +43,17 @@ spec:
   # rest of the service monitor
   endpoints:
   - relabelings:
-    - source_labels: [__meta_kubernetes_pod_label_app]
+    - sourceLabels: [__meta_kubernetes_pod_label_app]
       action: replace
       regex: (.*)
       replacement: $$1
-      target_label: double_dollar_sign
-    - source_labels: [__meta_kubernetes_pod_label_app]
+      targetLabel: double_dollar_sign
+    - sourceLabels: [__meta_kubernetes_pod_label_app]
       action: replace
       regex: (.*)
       replacement: $1
-      target_label: single_dollar_sign
-  - metricRelabelings:
+      targetLabel: single_dollar_sign
+    metricRelabelings:
       - action: keep
         regex: up
         sourceLabels: