ci/cd: Make Kafka tests isolated.

pvital · pvital · commit f1b7dd1dd75c · 2025-03-20T01:40:37.000-07:00
Signed-off-by: Paulo Vital &lt;paulo.vital@ibm.com&gt;
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -52,6 +52,9 @@ commands:
       gevent:
         default: ""
         type: string
+      kafka:
+        default: ""
+        type: string
       tests:
         default: "tests"
         type: string
@@ -61,6 +64,7 @@ commands:
           environment:
             CASSANDRA_TEST: "<<parameters.cassandra>>"
             GEVENT_STARLETTE_TEST: "<<parameters.gevent>>"
+            KAFKA_TEST: "<<parameters.kafka>>"
           command: |
             . venv/bin/activate
             coverage run --source=instana -m pytest -v --junitxml=test-results <<parameters.tests>>
@@ -136,15 +140,6 @@ jobs:
         environment:
           PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
           PUBSUB_PROJECT1: test-project,test-topic
-      - image: public.ecr.aws/bitnami/kafka:3.9.0
-        environment:
-          KAFKA_CFG_NODE_ID: 0
-          KAFKA_CFG_PROCESS_ROLES: controller,broker
-          KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
-          KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
-          KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
-          KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
-          KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
     working_directory: ~/repo
     steps:
       - checkout
@@ -173,15 +168,6 @@ jobs:
         environment:
           PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
           PUBSUB_PROJECT1: test-project,test-topic
-      - image: public.ecr.aws/bitnami/kafka:3.9.0
-        environment:
-          KAFKA_CFG_NODE_ID: 0
-          KAFKA_CFG_PROCESS_ROLES: controller,broker
-          KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
-          KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
-          KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
-          KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
-          KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
     working_directory: ~/repo
     steps:
       - checkout
@@ -210,15 +196,6 @@ jobs:
         environment:
           PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
           PUBSUB_PROJECT1: test-project,test-topic
-      - image: public.ecr.aws/bitnami/kafka:3.9.0
-        environment:
-          KAFKA_CFG_NODE_ID: 0
-          KAFKA_CFG_PROCESS_ROLES: controller,broker
-          KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
-          KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
-          KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
-          KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
-          KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
     working_directory: ~/repo
     steps:
       - checkout
@@ -248,15 +225,6 @@ jobs:
         environment:
           PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
           PUBSUB_PROJECT1: test-project,test-topic
-      - image: public.ecr.aws/bitnami/kafka:3.9.0
-        environment:
-          KAFKA_CFG_NODE_ID: 0
-          KAFKA_CFG_PROCESS_ROLES: controller,broker
-          KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
-          KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
-          KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
-          KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
-          KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
     working_directory: ~/repo
     steps:
       - checkout
@@ -286,15 +254,6 @@ jobs:
         environment:
           PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
           PUBSUB_PROJECT1: test-project,test-topic
-      - image: public.ecr.aws/bitnami/kafka:3.9.0
-        environment:
-          KAFKA_CFG_NODE_ID: 0
-          KAFKA_CFG_PROCESS_ROLES: controller,broker
-          KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
-          KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
-          KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
-          KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
-          KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
     working_directory: ~/repo
     steps:
       - checkout
@@ -338,15 +297,6 @@ jobs:
         environment:
           PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
           PUBSUB_PROJECT1: test-project,test-topic
-      - image: public.ecr.aws/bitnami/kafka:3.9.0
-        environment:
-          KAFKA_CFG_NODE_ID: 0
-          KAFKA_CFG_PROCESS_ROLES: controller,broker
-          KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
-          KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
-          KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
-          KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
-          KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
     working_directory: ~/repo
     steps:
       - checkout
@@ -376,15 +326,6 @@ jobs:
         environment:
           PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
           PUBSUB_PROJECT1: test-project,test-topic
-      - image: public.ecr.aws/bitnami/kafka:3.9.0
-        environment:
-          KAFKA_CFG_NODE_ID: 0
-          KAFKA_CFG_PROCESS_ROLES: controller,broker
-          KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
-          KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
-          KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
-          KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
-          KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
     working_directory: ~/repo
     steps:
       - checkout
@@ -443,6 +384,30 @@ jobs:
       - store-pytest-results
       - store-coverage-report
 
+  py312kafka:
+    docker:
+      - image: public.ecr.aws/docker/library/python:3.12
+      - image: public.ecr.aws/bitnami/kafka:3.9.0
+        environment:
+          KAFKA_CFG_NODE_ID: 0
+          KAFKA_CFG_PROCESS_ROLES: controller,broker
+          KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
+          KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
+          KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
+          KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
+          KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
+    working_directory: ~/repo
+    steps:
+      - checkout
+      - check-if-tests-needed
+      - pip-install-deps:
+          requirements: "tests/requirements-kafka.txt"
+      - run-tests-with-coverage-report:
+          kafka: "true"
+          tests: "tests/clients/kafka/test*.py"
+      - store-pytest-results
+      - store-coverage-report
+
 workflows:
   version: 2
   build:
@@ -457,6 +422,7 @@ workflows:
       - py39cassandra
       - py39gevent_starlette
       - py312aws
+      - py312kafka
       - final_job:
           requires:
             - python38
diff --git a/.tekton/pipeline.yaml b/.tekton/pipeline.yaml
@@ -87,3 +87,17 @@ spec:
       workspaces:
         - name: task-pvc
           workspace: python-tracer-ci-pipeline-pvc
+    - name: unittest-kafka
+      runAfter:
+        - clone
+      matrix:
+        params:
+          - name: imageDigest
+            value:
+              # public.ecr.aws/docker/library/python:3.12.9-bookworm
+              - "sha256:ae24158f83adcb3ec1dead14356e6debc9f3125167624408d95338faacc5cce3"
+      taskRef:
+        name: python-tracer-unittest-kafka-task
+      workspaces:
+        - name: task-pvc
+          workspace: python-tracer-ci-pipeline-pvc
diff --git a/.tekton/run_unittests.sh b/.tekton/run_unittests.sh
@@ -32,9 +32,13 @@ gevent_starlette)
 aws)
   export REQUIREMENTS='requirements.txt'
   export TESTS=('tests_aws') ;;
+kafka)
+  export REQUIREMENTS='requirements-kafka.txt'
+  export TESTS=('tests/clients/kafka')
+  export KAFKA_TEST='true' ;;
 *)
   echo "ERROR \$TEST_CONFIGURATION='${TEST_CONFIGURATION}' is unsupported " \
-       "not in (default|cassandra|gevent_starlette|aws)" >&2
+       "not in (default|cassandra|gevent_starlette|aws|kafka)" >&2
   exit 3 ;;
 esac
 
diff --git a/.tekton/task.yaml b/.tekton/task.yaml
@@ -131,25 +131,6 @@ spec:
     - name: rabbitmq
       # public.ecr.aws/docker/library/rabbitmq:3.13.0
       image: public.ecr.aws/docker/library/rabbitmq@sha256:39de1a4fc6c72d12bd5dfa23e8576536fd1c0cc8418344cd5a51addfc9a1145d
-    - name: kafka
-      # public.ecr.aws/bitnami/kafka:3.9.0
-      image: public.ecr.aws/bitnami/kafka@sha256:d2890d68f96b36da3c8413fa94294f018b2f95d87cf108cbf71eab510572d9be
-      env:
-        - name: KAFKA_CFG_NODE_ID
-          value: "0"
-        - name: KAFKA_CFG_PROCESS_ROLES
-          value: "controller,broker"
-        - name: KAFKA_CFG_LISTENERS
-          value: "PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094"
-        - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP
-          value: "CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT"
-        - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS
-          value: "0@kafka:9093"
-        - name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES
-          value: "CONTROLLER"
-        - name: KAFKA_CFG_ADVERTISED_LISTENERS
-          value: "PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094"
-    
   params:
   - name: imageDigest
     type: string
@@ -186,3 +167,43 @@ spec:
       workingDir: /workspace/python-sensor/
       command:
       - /workspace/python-sensor/.tekton/run_unittests.sh
+---
+apiVersion: tekton.dev/v1
+kind: Task
+metadata:
+  name: python-tracer-unittest-kafka-task
+spec:
+  sidecars:
+    - name: kafka
+      # public.ecr.aws/bitnami/kafka:3.9.0
+      image: public.ecr.aws/bitnami/kafka@sha256:d2890d68f96b36da3c8413fa94294f018b2f95d87cf108cbf71eab510572d9be
+      env:
+        - name: KAFKA_CFG_NODE_ID
+          value: "0"
+        - name: KAFKA_CFG_PROCESS_ROLES
+          value: "controller,broker"
+        - name: KAFKA_CFG_LISTENERS
+          value: "PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094"
+        - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP
+          value: "CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT"
+        - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS
+          value: "0@kafka:9093"
+        - name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES
+          value: "CONTROLLER"
+        - name: KAFKA_CFG_ADVERTISED_LISTENERS
+          value: "PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094"
+  params:
+  - name: imageDigest
+    type: string
+  workspaces:
+    - name: task-pvc
+      mountPath: /workspace
+  steps:
+    - name: unittest
+      image: public.ecr.aws/docker/library/python@$(params.imageDigest)
+      env:
+        - name: TEST_CONFIGURATION
+          value: kafka
+      workingDir: /workspace/python-sensor/
+      command:
+      - /workspace/python-sensor/.tekton/run_unittests.sh
diff --git a/tests/clients/kafka/test_confluent_kafka.py b/tests/clients/kafka/test_confluent_kafka.py
@@ -3,7 +3,11 @@
 from typing import Generator
 
 import pytest
-from confluent_kafka import Consumer, KafkaException, Producer  # noqa: F401
+from confluent_kafka import (
+    Consumer,
+    KafkaException,
+    Producer,
+)
 from confluent_kafka.admin import AdminClient, NewTopic
 from opentelemetry.trace import SpanKind
 
@@ -70,7 +74,7 @@ def test_trace_confluent_kafka_produce(self) -> None:
         assert kafka_span.n == "kafka"
         assert kafka_span.k == SpanKind.CLIENT
         assert kafka_span.data["kafka"]["service"] == testenv["kafka_topic"]
-        assert kafka_span.data["kafka"]["access"] == "send"
+        assert kafka_span.data["kafka"]["access"] == "produce"
 
     def test_trace_confluent_kafka_consume(self) -> None:
         # Produce some events
@@ -159,8 +163,7 @@ def test_trace_confluent_kafka_error(self) -> None:
         consumer.subscribe(["inexistent_kafka_topic"])
 
         with tracer.start_as_current_span("test"):
-            msg = consumer.poll(timeout=5)  # noqa: F841
-            # assert not msg
+            consumer.consume(-10)
 
         consumer.close()
 
@@ -183,8 +186,8 @@ def test_trace_confluent_kafka_error(self) -> None:
         assert kafka_span.n == "kafka"
         assert kafka_span.k == SpanKind.SERVER
         assert not kafka_span.data["kafka"]["service"]
-        assert kafka_span.data["kafka"]["access"] == "poll"
+        assert kafka_span.data["kafka"]["access"] == "consume"
         assert (
             kafka_span.data["kafka"]["error"]
-            == "'NoneType' object has no attribute 'topic'"
+            == "num_messages must be between 0 and 1000000 (1M)"
         )
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -17,12 +17,12 @@
 
 from instana.agent.host import HostAgent
 from instana.collector.base import BaseCollector
+from instana.fsm import TheMachine
 from instana.recorder import StanRecorder
 from instana.span.base_span import BaseSpan
 from instana.span.span import InstanaSpan
 from instana.span_context import SpanContext
 from instana.tracer import InstanaTracerProvider
-from instana.fsm import TheMachine
 
 collect_ignore_glob = [
     "*test_gevent*",
@@ -42,6 +42,8 @@
     collect_ignore_glob.append("*test_gevent*")
     collect_ignore_glob.append("*test_starlette*")
 
+if not os.environ.get("KAFKA_TEST"):
+    collect_ignore_glob.append("*kafka/test*")
 
 if sys.version_info >= (3, 13):
     # Currently not installable dependencies because of 3.13 incompatibilities
diff --git a/tests/requirements-kafka.txt b/tests/requirements-kafka.txt
@@ -0,0 +1,6 @@
+coverage>=5.5
+mock>=2.0.0
+pytest
+kafka-python>=2.0.0; python_version < "3.12"
+kafka-python-ng>=2.0.0; python_version >= "3.12"
+confluent-kafka>=2.0.0
diff --git a/tests/requirements-pre314.txt b/tests/requirements-pre314.txt
@@ -41,5 +41,3 @@ tornado>=6.4.1
 uvicorn>=0.13.4
 urllib3>=1.26.5
 httpx>=0.27.0
-kafka-python-ng>=2.0.0
-confluent-kafka>=2.0.0
diff --git a/tests/requirements.txt b/tests/requirements.txt
@@ -40,6 +40,3 @@ tornado>=6.4.1
 uvicorn>=0.13.4
 urllib3>=1.26.5
 httpx>=0.27.0
-kafka-python>=2.0.0; python_version < "3.12"
-kafka-python-ng>=2.0.0; python_version >= "3.12"
-confluent-kafka>=2.0.0
