Merge branch 'master' into seemant-parachain-umbrella-crate-mig

Author: seemantaggarwal

.config/taplo.toml

@@ -15,7 +15,7 @@ exclude = [
 [formatting]
 reorder_arrays = true
 inline_table_expand = false
-array_auto_expand = false
+array_auto_expand = true
 array_auto_collapse = false
 indent_string = "	"         # tab
 

.github/scripts/common/lib.sh

@@ -243,17 +243,17 @@ fetch_release_artifacts() {
 # - REPO in the form paritytech/polkadot
 fetch_debian_package_from_s3() {
   BINARY=$1
-  echo "Version    : $VERSION"
+  echo "Version    : $NODE_VERSION"
   echo "Repo       : $REPO"
   echo "Binary     : $BINARY"
-  echo "Tag        : $RELEASE_TAG"
+  echo "Tag        : $VERSION"
   OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
   echo "OUTPUT_DIR : $OUTPUT_DIR"
 
   URL_BASE=$(get_s3_url_base $BINARY)
   echo "URL_BASE=$URL_BASE"
 
-  URL=$URL_BASE/$RELEASE_TAG/x86_64-unknown-linux-gnu/${BINARY}_${VERSION}_amd64.deb
+  URL=$URL_BASE/$VERSION/x86_64-unknown-linux-gnu/${BINARY}_${NODE_VERSION}_amd64.deb
 
   mkdir -p "$OUTPUT_DIR"
   pushd "$OUTPUT_DIR" > /dev/null

.github/scripts/release/release_lib.sh

@@ -71,7 +71,7 @@ git_show_log() {
 # 1_012_000 or 1_012_001 if SUFFIX is set
 function get_spec_version() {
     INPUT=$1
-    SUFFIX=${SUFFIX:-000} #this variable makes it possible to set a specific ruuntime version like 93826 it can be intialised as sestem variable
+    SUFFIX=${SUFFIX:-000} #this variable makes it possible to set a specific runtime version like 93826 it can be initialised as system variable
     [[ $INPUT =~ .*([0-9]+\.[0-9]+\.[0-9]{1,2}).* ]]
     VERSION="${BASH_REMATCH[1]}"
     MATCH="${BASH_REMATCH[0]}"

.github/workflows/build-publish-images.yml

@@ -266,6 +266,9 @@ jobs:
           path: artifacts.tar
           retention-days: 1
 
+
+  ### Build zombienet test artifacts ########################
+
   #
   #
   #
@@ -296,6 +299,66 @@ jobs:
           path: artifacts.tar
           retention-days: 1
 
+  #
+  #
+  #
+  prepare-polkadot-zombienet-artifacts:
+    needs: [preflight]
+    runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    timeout-minutes: 60
+    container:
+      image: ${{ needs.preflight.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: build
+        run: |
+          forklift cargo nextest --manifest-path polkadot/zombienet-sdk-tests/Cargo.toml archive --locked --features zombie-metadata --archive-file polkadot-zombienet-tests.tar.zst
+      - name: pack artifacts
+        run: |
+          mkdir -p artifacts
+          cp polkadot-zombienet-tests.tar.zst ./artifacts
+
+      - name: tar
+        run: tar -cvf artifacts.tar artifacts
+
+      - name: upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.job }}-${{ needs.preflight.outputs.SOURCE_REF_SLUG }}
+          path: artifacts.tar
+          retention-days: 1
+
+  #
+  #
+  #
+  prepare-cumulus-zombienet-artifacts:
+    needs: [preflight]
+    runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    timeout-minutes: 60
+    container:
+      image: ${{ needs.preflight.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: build
+        run: |
+          forklift cargo nextest --manifest-path cumulus/zombienet/zombienet-sdk/Cargo.toml archive --locked --features zombie-ci --archive-file cumulus-zombienet-tests.tar.zst
+      - name: pack artifacts
+        run: |
+          mkdir -p artifacts
+          cp cumulus-zombienet-tests.tar.zst ./artifacts
+
+      - name: tar
+        run: tar -cvf artifacts.tar artifacts
+
+      - name: upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.job }}-${{ needs.preflight.outputs.SOURCE_REF_SLUG }}
+          path: artifacts.tar
+          retention-days: 1
+
   ### Publish ########################
 
   #

.github/workflows/checks-quick.yml

@@ -102,7 +102,6 @@ jobs:
           --exclude
           "substrate/frame/contracts/fixtures/build"
           "substrate/frame/contracts/fixtures/contracts/common"
-          "substrate/frame/revive/fixtures/contracts/common"
       - name: deny git deps
         run: python3 .github/scripts/deny-git-deps.py .
   check-markdown:

.github/workflows/release-11_rc-automation.yml

@@ -1,6 +1,6 @@
 name: Release - RC automation
 on:
-  # TODO: Activate it and delete old branches patterns, when the release process from stable is setteled
+  # TODO: Activate it and delete old branches patterns, when the release process from stable is settled
   #push:
     # branches:
     #   # Catches release-polkadot-v1.2.3, release-v1.2.3-rc1, etc

.github/workflows/release-30_publish_release_draft.yml

@@ -39,17 +39,6 @@ jobs:
           RELEASE_TAG=$(validate_stable_tag ${{ inputs.release_tag }})
           echo "release_tag=${RELEASE_TAG}" >> $GITHUB_OUTPUT
 
-  get-rust-versions:
-    needs: [ validate-inputs ]
-    runs-on: ubuntu-latest
-    outputs:
-      rustc-stable: ${{ steps.get-rust-versions.outputs.stable }}
-    steps:
-      - id: get-rust-versions
-        run: |
-          RUST_STABLE_VERSION=$(curl -sS https://raw.githubusercontent.com/paritytech/scripts/master/dockerfiles/ci-unified/Dockerfile | grep -oP 'ARG RUST_STABLE_VERSION=\K[^ ]+')
-          echo "stable=$RUST_STABLE_VERSION" >> $GITHUB_OUTPUT
-
   build-runtimes:
     needs: [ validate-inputs ]
     uses: "./.github/workflows/release-srtool.yml"
@@ -65,7 +54,7 @@ jobs:
   publish-release-draft:
     runs-on: ubuntu-latest
     environment: release
-    needs: [ validate-inputs, get-rust-versions, build-runtimes ]
+    needs: [ validate-inputs, build-runtimes ]
     outputs:
       release_url: ${{ steps.create-release.outputs.html_url }}
       asset_upload_url: ${{ steps.create-release.outputs.upload_url }}
@@ -86,7 +75,6 @@ jobs:
       - name: Prepare draft
         id: draft
         env:
-          RUSTC_STABLE: ${{ needs.get-rust-versions.outputs.rustc-stable }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           ASSET_HUB_WESTEND_DIGEST: ${{ github.workspace}}/asset-hub-westend-runtime/asset-hub-westend-srtool-digest.json
           BRIDGE_HUB_WESTEND_DIGEST: ${{ github.workspace}}/bridge-hub-westend-runtime/bridge-hub-westend-srtool-digest.json
@@ -100,6 +88,8 @@ jobs:
         run: |
           . ./.github/scripts/common/lib.sh
 
+          export RUSTC_STABLE=$(grep -oP '(?<=-)[0-9]+\.[0-9]+\.[0-9]+(?=-)' .github/env)
+
           export REF1=$(get_latest_release_tag)
           if [[ -z "$RELEASE_TAG" ]]; then
             export REF2="${{ github.ref_name }}"

.github/workflows/release-40_publish-deb-package.yml

@@ -1,10 +1,10 @@
-name: Release - Publish polakdot deb package
+name: Release - Publish polkadot deb package
 
 on:
   workflow_dispatch:
     inputs:
       tag:
-        description: Current final release tag in the format polakdot-stableYYMM or polkadot-stable-YYMM-X
+        description: Current final release tag in the format polkadot-stableYYMM or polkadot-stable-YYMM-X
         default: polkadot-stable2412
         required: true
         type: string
@@ -44,9 +44,9 @@ jobs:
     needs: [validate-inputs]
     env:
       REPO: ${{ github.repository }}
-      RELEASE_TAG: ${{ needs.validate-inputs.outputs.release_tag }}
+      VERSION: ${{ needs.validate-inputs.outputs.release_tag }}
     outputs:
-      VERSION: ${{ steps.fetch_artifacts_from_s3.outputs.VERSION }}
+      NODE_VERSION: ${{ steps.fetch_artifacts_from_s3.outputs.NODE_VERSION }}
 
     steps:
       - name: Checkout sources
@@ -57,8 +57,8 @@ jobs:
         run: |
           . ./.github/scripts/common/lib.sh
 
-          VERSION="$(get_polkadot_node_version_from_code)"
-          echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
+          NODE_VERSION="$(get_polkadot_node_version_from_code)"
+          echo "NODE_VERSION=${NODE_VERSION}" >> $GITHUB_OUTPUT
 
           fetch_debian_package_from_s3 polkadot
 
@@ -76,7 +76,7 @@ jobs:
       AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
       AWS_DEB_PATH: "s3://releases-package-repos/deb"
       LOCAL_DEB_REPO_PATH: ${{ github.workspace }}/deb
-      VERSION: ${{ needs.fetch-artifacts-from-s3.outputs.VERSION }}
+      NODE_VERSION: ${{ needs.fetch-artifacts-from-s3.outputs.NODE_VERSION }}
 
     steps:
       - name: Install pgpkkms
@@ -135,7 +135,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
           # Add the new deb to the repo
-          reprepro -b "$LOCAL_DEB_REPO_PATH" includedeb "${{ inputs.distribution }}" "release-artifacts/polkadot_${VERSION}_amd64.deb"
+          reprepro -b "$LOCAL_DEB_REPO_PATH" includedeb "${{ inputs.distribution }}" "release-artifacts/polkadot_${NODE_VERSION}_amd64.deb"
 
       - name: Upload updated deb repo
         env:

.github/workflows/release-50_publish-docker.yml

@@ -281,7 +281,7 @@ jobs:
         uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0
 
       - name: Cache Docker layers
-        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}

.github/workflows/release-reusable-promote-to-final.yml

@@ -35,6 +35,9 @@ jobs:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
       AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      RELEASE_TAG: ${{ inputs.release_tag }}
+      PACKAGE: ${{ inputs.package }}
+      TARGET: ${{ inputs.target }}
 
     steps:
       - name: Checkout sources
@@ -44,22 +47,25 @@ jobs:
         id: prepare_final_tag
         shell: bash
         run: |
-          tag="$(echo ${{ inputs.release_tag }} | sed 's/-rc[0-9]*$//')"
+          tag="$(echo $RELEASE_TAG | sed 's/-rc[0-9]*$//')"
           echo $tag
           echo "FINAL_TAG=${tag}" >> $GITHUB_OUTPUT
 
       - name: Fetch binaries from s3 based on version
         run: |
           . ./.github/scripts/common/lib.sh
 
-          VERSION="${{ inputs.release_tag }}"
-          if [[ ${{ inputs.package }} == 'polkadot' ]]; then
+          VERSION="$RELEASE_TAG"
+          if [[ "$PACKAGE" == 'polkadot' ]]; then
             packages=(polkadot polkadot-prepare-worker polkadot-execute-worker)
             for package in "${packages[@]}"; do
-              fetch_release_artifacts_from_s3 $package ${{ inputs.target }}
+              OUTPUT_DIR="./release-artifacts/$TARGET/${package}"
+              fetch_release_artifacts_from_s3 "$package" "$TARGET"
             done
+            NODE_VERSION="$(get_polkadot_node_version_from_code)"
+            fetch_debian_package_from_s3 polkadot
           else
-            fetch_release_artifacts_from_s3 ${{ inputs.package }} ${{ inputs.target }}
+            fetch_release_artifacts_from_s3 "$PACKAGE" "$TARGET"
           fi
 
       - name: Configure AWS Credentials
@@ -73,11 +79,11 @@ jobs:
         run: |
           . ./.github/scripts/release/release_lib.sh
 
-          if [[ ${{ inputs.package }} == 'polkadot' ]]; then
+          if [[ "$PACKAGE" == 'polkadot' ]]; then
             packages=(polkadot polkadot-prepare-worker polkadot-execute-worker)
             for package in "${packages[@]}"; do
               upload_s3_release $package ${{ steps.prepare_final_tag.outputs.final_tag }} ${{ inputs.target }}
             done
           else
-            upload_s3_release ${{ inputs.package }} ${{ steps.prepare_final_tag.outputs.final_tag }} ${{ inputs.target }}
+            upload_s3_release "$PACKAGE" ${{ steps.prepare_final_tag.outputs.final_tag }} "$TARGET"
           fi

.github/workflows/release-reusable-rc-buid.yml

@@ -15,7 +15,7 @@ on:
         type: string
 
       release_tag:
-        description: Tag matching the actual release candidate with the format stableYYMM-rcX or stableYYMM
+        description: Tag matching the actual release candidate with the format polkadpt-stableYYMM(-rcX) or plkadot-stableYYMM-X(-rcX)
         required: true
         type: string
 

.github/workflows/reusable-preflight.yml

@@ -78,8 +78,8 @@ jobs:
   preflight:
     runs-on: ubuntu-latest
     outputs:
-      changes_rust: ${{ steps.set_changes.outputs.rust_any_changed || steps.set_changes.outputs.currentWorkflow_any_changed }}
-      changes_currentWorkflow: ${{ steps.set_changes.outputs.currentWorkflow_any_changed }}
+      changes_rust: true
+      changes_currentWorkflow: true
 
       IMAGE: ${{ steps.set_image.outputs.IMAGE }}
 
@@ -112,19 +112,20 @@ jobs:
           echo "currentWorkflowFile=$(echo ${{ github.workflow_ref }} | sed -nE "s/.*(\.github\/workflows\/[a-zA-Z0-9_-]*\.y[a]?ml)@refs.*/\1/p")" >> $GITHUB_OUTPUT
           echo "currentActionDir=$(echo ${{ github.action_path }} | sed -nE "s/.*(\.github\/actions\/[a-zA-Z0-9_-]*)/\1/p")" >> $GITHUB_OUTPUT
 
-      - name: Set changes
-        id: set_changes
-        uses: tj-actions/changed-files@v45
-        with:
-          files_yaml: |
-            rust:
-              - '**/*'
-              - '!.github/**/*'
-              - '!prdoc/**/*'
-              - '!docs/**/*'
-            currentWorkflow:
-              - '${{ steps.current_file.outputs.currentWorkflowFile }}'
-              - '.github/workflows/reusable-preflight.yml'
+      # removed due to https://news.ycombinator.com/item?id=43368870
+      #- name: Set changes
+      #  id: set_changes
+      #  uses: tj-actions/changed-files@v45
+      #  with:
+      #    files_yaml: |
+      #      rust:
+      #        - '**/*'
+      #        - '!.github/**/*'
+      #        - '!prdoc/**/*'
+      #        - '!docs/**/*'
+      #      currentWorkflow:
+      #        - '${{ steps.current_file.outputs.currentWorkflowFile }}'
+      #        - '.github/workflows/reusable-preflight.yml'
 
       #
       # Set image
@@ -180,7 +181,6 @@ jobs:
         shell: bash
         run: |
           echo "workflow file: ${{ steps.current_file.outputs.currentWorkflowFile }}"
-          echo "Modified: ${{ steps.set_changes.outputs.modified_keys }}"
 
   #
   #