Ubuntu 24.04 and arm64 linux package verification (confluentinc#4954)

* Verify Ubuntu 24.04 and arm64 packages
* Add Semaphore task for verifying

Author: emasab

.semaphore/verify-linux-packages.yml

@@ -0,0 +1,41 @@
+version: v1.0
+name: verify-linux-packages
+
+agent:
+  machine:
+    type: s1-prod-ubuntu22-04-amd64-1
+
+execution_time_limit:
+  hours: 2
+
+global_job_config:
+  prologue:
+    commands:
+      - checkout
+  env_vars:
+    - name: REPOSITORY_BASE_URL
+      value: "https://packages.confluent.io"
+
+blocks:
+  - name: "Verify Linux packages (x86_64)"
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-ubuntu22-04-amd64-2
+      jobs:
+        - name: "Verify packages"
+          commands:
+            - cd packaging/cp
+            - ./verify-packages.sh ${REPOSITORY_BASE_URL} linux/amd64 ${LIBRDKAFKA_VERSION}
+  - name: "Verify Linux packages (aarch64)"
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-ubuntu22-04-arm64-2
+      jobs:
+        - name: "Verify packages"
+          commands:
+            - cd packaging/cp
+            - ./verify-packages.sh ${REPOSITORY_BASE_URL} linux/arm64 ${LIBRDKAFKA_VERSION}

packaging/cp/README.md

@@ -4,10 +4,13 @@ This small set of scripts verifies the librdkafka packages that
 are part of the Confluent Platform.
 
 The base_url is the http S3 bucket path to the a PR job, or similar.
+Pass the platform (e.g. linux/arm64) and the expected librdkafka version too.
 
 ## How to use
 
-    $ ./verify-packages.sh 7.6 https://packages.confluent.io
+```
+$ ./verify-packages.sh https://packages.confluent.io linux/amd64 2.8.0
+```
 
 Requires docker and patience.
 

packaging/cp/check_features.c

@@ -15,12 +15,18 @@ int main(int argc, char **argv) {
         printf("librdkafka %s (0x%x, define: 0x%x)\n", rd_kafka_version_str(),
                rd_kafka_version(), RD_KAFKA_VERSION);
 
-        if (argc > 1 && !(argc & 1)) {
-                printf("Usage: %s [config.property config-value ..]\n",
+        if (argc < 2 || (argc > 2 && (argc & 1))) {
+                printf("Usage: %s  <version_string> [config.property config-value ..]\n",
                        argv[0]);
                 return 1;
         }
 
+        if (!strstr(rd_kafka_version_str(), argv[1])) {
+                printf("ERROR: version string mismatch: expected %s, got %s\n",
+                       argv[1], rd_kafka_version_str());
+                return 1;
+        }
+
         conf = rd_kafka_conf_new();
         res  = rd_kafka_conf_get(conf, "builtin.features", buf, &sz);
 
@@ -46,7 +52,7 @@ int main(int argc, char **argv) {
         printf("all expected features matched: %s\n", expected_features);
 
         /* Apply config from argv key value pairs */
-        for (i = 1; i + 1 < argc; i += 2) {
+        for (i = 2; i + 1 < argc; i += 2) {
                 printf("verifying config %s=%s\n", argv[i], argv[i + 1]);
                 if (rd_kafka_conf_set(conf, argv[i], argv[i + 1], errstr,
                                       sizeof(errstr)) != RD_KAFKA_CONF_OK) {

packaging/cp/verify-deb.sh

@@ -3,30 +3,30 @@
 
 set -e
 
-cpver=$1
-base_url=$2
+base_url=$1
+version=$2
 
 if [[ -z $base_url ]]; then
-    echo "Usage: $0 <cp-base-ver> <base_url>"
+    echo "Usage: $0 <base_url>"
     exit 1
 fi
 
 apt-get update
 apt-get install -y apt-transport-https wget gnupg2 lsb-release
 
-wget -qO - ${base_url}/deb/${cpver}/archive.key | apt-key add -
+wget -qO - ${base_url}/clients/deb/archive.key | apt-key add -
 
 release=$(lsb_release -cs)
 cat >/etc/apt/sources.list.d/Confluent.list <<EOF
-deb [arch=amd64] $base_url/clients/deb ${release} main
+deb $base_url/clients/deb ${release} main
 EOF
 
 apt-get update
 apt-get install -y librdkafka-dev gcc
 
 gcc /v/check_features.c -o /tmp/check_features -lrdkafka
 
-/tmp/check_features
+/tmp/check_features $version
 
 # FIXME: publish plugins in newer versions
 # apt-get install -y confluent-librdkafka-plugins

packaging/cp/verify-packages.sh

@@ -2,47 +2,70 @@
 #
 # Verifies RPM and DEB packages from Confluent Platform
 #
+base_url=$1
+platform=$2
+version=$3
 
-cpver=$1
-base_url=$2
-
-if [[ -z $base_url ]]; then
-    echo "Usage: $0 <CP-M.m-version> <base-url>"
+if [[ -z $base_url || -z $version ]]; then
+    echo "Usage: $0 <base-url> <platform> <version>"
     echo ""
-    echo " <CP-M.m-version> is the Major.minor version of CP, e.g., 5.3"
     echo " <base-url> is the release base bucket URL"
+    echo " <platform> is platform to verify (e.g. linux/amd64)"
+    echo " <version> is the expected version"
     exit 1
 fi
 
 thisdir="$( cd "$(dirname "$0")" ; pwd -P )"
 
-echo "#### Verifying RPM packages ####"
-docker run -v $thisdir:/v rockylinux:8 /v/verify-rpm.sh $cpver $base_url
-docker run -v $thisdir:/v rockylinux:9 /v/verify-rpm.sh $cpver $base_url
-rpm_status=$?
-
-echo "#### Verifying Debian packages ####"
-docker run -v $thisdir:/v debian:10 /v/verify-deb.sh $cpver $base_url
-docker run -v $thisdir:/v debian:11 /v/verify-deb.sh $cpver $base_url
-docker run -v $thisdir:/v debian:12 /v/verify-deb.sh $cpver $base_url
-docker run -v $thisdir:/v ubuntu:20.04 /v/verify-deb.sh $cpver $base_url
-docker run -v $thisdir:/v ubuntu:22.04 /v/verify-deb.sh $cpver $base_url
-deb_status=$?
+verify_debian() {
+    local version=$2
+    docker run -v $thisdir:/v $1 /v/verify-deb.sh $base_url $version
+    deb_status=$?
+    if [[ $deb_status == 0 ]]; then
+        echo "SUCCESS: Debian based $1 $2 packages verified"
+    else
+        echo "ERROR: Debian based $1 $2 package verification failed"
+        exit 1
+    fi
+}
 
+verify_rpm() {
+    local version=$2
+    docker run -v $thisdir:/v $1 /v/verify-rpm.sh $base_url $version
+    rpm_status=$?
+    if [[ $rpm_status == 0 ]]; then
+        echo "SUCCESS: RPM $1 $2 packages verified"
+    else
+        echo "ERROR: RPM $1 $2 package verification failed"
+        exit 1
+    fi
+}
 
-if [[ $rpm_status == 0 ]]; then
-    echo "SUCCESS: RPM packages verified"
-else
-    echo "ERROR: RPM package verification failed"
-fi
+verify_rpm_distros() {
+    local platform=$1
+    local version=$2
+    echo "#### Verifying RPM packages for $platform ####"
+    # Last RHEL 8 version is 2.4.0
+    verify_rpm rockylinux:8 "2.4.0"
+    verify_rpm rockylinux:9 $version
+}
 
-if [[ $deb_status == 0 ]]; then
-    echo "SUCCESS: Debian packages verified"
-else
-    echo "ERROR: Debian package verification failed"
-fi
+verify_debian_distros() {
+    local platform=$1
+    local version=$2
+    echo "#### Verifying Debian packages for $platform ####"
+    # Last Debian 10 version is 2.5.0
+    verify_debian debian:10 "2.5.0"
+    verify_debian debian:11 $version
+    verify_debian debian:12 $version
+    verify_debian ubuntu:20.04 $version
+    verify_debian ubuntu:22.04 $version
+    verify_debian ubuntu:24.04 $version
+}
 
-if [[ $deb_status != 0 || $rpm_status != 0 ]]; then
-    exit 1
-fi
+verify_distros() {
+    verify_rpm_distros $1 $2
+    verify_debian_distros $1 $2
+}
 
+verify_distros $platform $version

packaging/cp/verify-rpm.sh

@@ -3,22 +3,15 @@
 
 set -e
 
-cpver=$1
-base_url=$2
+base_url=$1
+version=$2
 
 if [[ -z $base_url ]]; then
-    echo "Usage: $0 <cp-base-ver> <base_url>"
+    echo "Usage: $0 <base_url>"
     exit 1
 fi
 
 cat >/etc/yum.repos.d/Confluent.repo <<EOF
-[Confluent]
-name=Confluent repository
-baseurl=$base_url/rpm/${cpver}
-gpgcheck=1
-gpgkey=$base_url/rpm/${cpver}/archive.key
-enabled=1
-
 [Confluent-Clients]
 name=Confluent Clients repository
 baseurl=$base_url/clients/rpm/centos/\$releasever/\$basearch
@@ -31,9 +24,9 @@ yum install -y librdkafka-devel gcc
 
 gcc /v/check_features.c -o /tmp/check_features -lrdkafka
 
-/tmp/check_features
+/tmp/check_features $version
 
 # Verify plugins
-yum install -y confluent-librdkafka-plugins
+# yum install -y confluent-librdkafka-plugins
 
-/tmp/check_features plugin.library.paths monitoring-interceptor
+# /tmp/check_features plugin.library.paths monitoring-interceptor

service.yml

@@ -16,3 +16,12 @@ semaphore:
   - /semaphore.*/
   - /dev_.*/
   - /feature\/.*/
+  tasks:
+  - name: verify-linux-packages
+    branch: master
+    scheduled: false
+    pipeline_file: .semaphore/verify-linux-packages.yml
+    parameters:
+      - name: LIBRDKAFKA_VERSION
+        required: true
+        description: librdkafka version to verify