Merge branch 'master' into sv--cus-2468-data-products-cache-issue

Author: anshbansal

.github/.codecov.yml

@@ -0,0 +1,65 @@
+comment:
+  layout: "header, files, footer" # remove "new" from "header" and "footer"
+  hide_project_coverage: true # set to false
+  require_changes: false  # if true: only post the comment if coverage changes
+
+codecov:
+  #due to ci-optimization, reports for modules that have not changed may be quite old
+  max_report_age: off
+
+flag_management:
+  default_rules: # the rules that will be followed for any flag added, generally
+    carryforward: true
+    statuses:
+      - type: project
+        target: auto
+        threshold: 0%  #Not enforcing project coverage yet.
+      - type: patch
+        target: 90%
+  individual_flags: # exceptions to the default rules above, stated flag by flag
+    - name: frontend
+      paths:
+        - "datahub-frontend/**"
+        - "datahub-web-react/**"
+    - name: backend
+      paths:
+        - "metadata-models/**"
+        - "datahub-upgrade/**"
+        - "entity-registry/**"
+        - "li-utils/**"
+        - "metadata-auth/**"
+        - "metadata-dao-impl/**"
+        - "metadata-events/**"
+        - "metadata-jobs/**"
+        - "metadata-service/**"
+        - "metadata-utils/**"
+        - "metadata-operation-context/**"
+        - "datahub-graphql-core/**"
+    - name: metadata-io
+      paths:
+        - "metadata-io/**"
+    - name: ingestion
+      paths:
+        - "metadata-ingestion/**"
+    - name: ingestion-airflow
+      paths:
+        - "metadata-ingestion-modules/airflow-plugin/**"
+    - name: ingestion-dagster
+      paths:
+        - "metadata-ingestion-modules/dagster-plugin/**"
+    - name: ingestion-gx-plugin
+      paths:
+        - "metadata-ingestion-modules/gx-plugin/**"
+    - name: ingestion-prefect
+      paths:
+        - "metadata-ingestion-modules/prefect-plugin/**"
+coverage:
+  status:
+    project:
+      default:
+        target: 0% # no threshold enforcement yet
+        only_pulls: true
+    patch:
+      default:
+        target: 90% # for new code added in the patch
+        only_pulls: true

.github/actions/ci-optimization/action.yml

@@ -1,5 +1,5 @@
-name: 'Identify CI Optimizations'
-description: 'Determine if code changes are specific to certain modules.'
+name: "Identify CI Optimizations"
+description: "Determine if code changes are specific to certain modules."
 
 outputs:
   frontend-only:
@@ -13,16 +13,16 @@ outputs:
     value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'true' }}
   backend-change:
     description: "Backend code has changed"
-    value: ${{ steps.filter.outputs.backend == 'true' }}
+    value: ${{ steps.filter.outputs.backend == 'true' ||  steps.trigger.outputs.trigger == 'manual' }}
   ingestion-change:
     description: "Ingestion code has changed"
-    value: ${{ steps.filter.outputs.ingestion == 'true' }}
+    value: ${{ steps.filter.outputs.ingestion == 'true' || steps.trigger.outputs.trigger == 'manual' }}
   ingestion-base-change:
     description: "Ingestion base image docker image has changed"
     value: ${{ steps.filter.outputs.ingestion-base == 'true' }}
   frontend-change:
     description: "Frontend code has changed"
-    value: ${{ steps.filter.outputs.frontend == 'true' }}
+    value: ${{ steps.filter.outputs.frontend == 'true' || steps.trigger.outputs.trigger == 'manual' }}
   docker-change:
     description: "Docker code has changed"
     value: ${{ steps.filter.outputs.docker == 'true' }}
@@ -44,27 +44,35 @@ outputs:
 runs:
   using: "composite"
   steps:
-    - uses: dorny/paths-filter@v2
+    - name: Check trigger type
+      id: trigger  # Add an ID to reference this step
+      shell: bash
+      run: |
+        if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          echo "trigger=manual" >> $GITHUB_OUTPUT
+        else
+          echo "trigger=pr" >> $GITHUB_OUTPUT
+        fi
+    - uses: dorny/paths-filter@v3
       id: filter
       with:
+        token: "" # Empty token forces it to use raw git commands.
         filters: |
           frontend:
             - "datahub-frontend/**"
             - "datahub-web-react/**"
-            - "smoke-test/tests/cypress/**"
             - "docker/datahub-frontend/**"
           ingestion:
             - "metadata-ingestion-modules/**"
             - "metadata-ingestion/**"
             - "metadata-models/**"
-            - "smoke-test/**"
-            - "docker/datahub-ingestion**"
+            - "docker/datahub-ingestion-base/**"
+            - "docker/datahub-ingestion/**"
           ingestion-base:
             - "docker/datahub-ingestion-base/**"
           docker:
             - "docker/**"
           backend:
-            - ".github/**"
             - "metadata-models/**"
             - "datahub-upgrade/**"
             - "entity-registry/**"
@@ -78,7 +86,6 @@ runs:
             - "metadata-utils/**"
             - "metadata-operation-context/**"
             - "datahub-graphql-core/**"
-            - "smoke-test/**"
             - "docker/**"
           kafka-setup:
             - "docker/kafka-setup/**"

.github/actions/docker-custom-build-and-push/action.yml

@@ -26,36 +26,59 @@ inputs:
   build-args:
     description: "List of build-time variables. Same as docker/build-push-action"
     required: false
-  tags:
-    # e.g. latest,head,sha12345
-    description: "List of tags to use for the Docker image"
+  image_tag:
+    # e.g. pr12345 OR head OR v0.1.2.3
+    description: "Main tag to use for the Docker image"
     required: true
+  flavor:
+    description: "Image flavor (e.g., slim, full)"
+    required: false
   target:
     description: "Sets the target stage to build"
     required: false
+  depot-project:
+    # Setting this will use native arm64 docker builds instead of QEMU emulation.
+    # This speeds up builds by 2-3x.
+    description: "Depot project id"
+    required: false
+
 outputs:
   image_tag:
     description: "Docker image tags"
     value: ${{ steps.docker_meta.outputs.tags }}
-  # image_name: ${{ env.DATAHUB_GMS_IMAGE }}
 
 runs:
   using: "composite"
 
   steps:
     - name: Docker meta
       id: docker_meta
-      uses: crazy-max/ghaction-docker-meta@v1
+      uses: docker/metadata-action@v5
       with:
-        # list of Docker images to use as base name for tags
         images: ${{ inputs.images }}
-        # add git short SHA as Docker tag
-        tag-custom: ${{ inputs.tags }}
-        tag-custom-only: true
+        flavor: |
+          latest=false
+        tags: |
+          type=raw,value=${{ inputs.image_tag }}
+          type=raw,value=head,suffix=${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }},enable={{is_default_branch}}
+          type=sha,prefix=,format=short,suffix=${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }}
+
+    - name: Single Tag
+      id: single_tag
+      shell: bash
+      run: |
+        IMAGES="""
+        ${{ inputs.images }}
+        """
+        TAGS="""
+        ${{ inputs.image_tag }}
+        """
+        echo "SINGLE_IMAGE=$(echo $IMAGES | tr '\n' ' ' | awk -F' |,' '{ print $1 }')" >> "$GITHUB_OUTPUT"
+        echo "SINGLE_TAG=$(echo $IMAGES | tr '\n' ' ' | awk -F' |,' '{ print $1 }'):$(echo $TAGS | tr '\n' ' ' | awk -F' |,' '{ print $1 }')" >> "$GITHUB_OUTPUT"
 
     # Code for testing the build when not pushing to Docker Hub.
     - name: Build and Load image for testing (if not publishing)
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       if: ${{ inputs.publish != 'true' }}
       with:
         context: ${{ inputs.context }}
@@ -68,39 +91,57 @@ runs:
         target: ${{ inputs.target }}
         load: true
         push: false
-        cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
-        cache-to: type=inline
-    - name: Single Tag
-      if: ${{ inputs.publish != 'true' }}
-      shell: bash
-      run: |
-        TAGS="""
-        ${{ steps.docker_meta.outputs.tags }}
-        """
-        echo "SINGLE_TAG=$(echo $TAGS | tr '\n' ' ' | awk -F' ' '{ print $1 }')" >> $GITHUB_OUTPUT
-      id: single_tag
+        cache-from: |
+          type=registry,ref=${{ steps.single_tag.outputs.SINGLE_IMAGE }}:head${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }}
+          type=registry,ref=${{ steps.docker_meta.outputs.tags }}
+        cache-to: |
+          type=inline
     - name: Upload image locally for testing (if not publishing)
-      uses: ishworkh/docker-image-artifact-upload@v1
+      uses: ishworkh/container-image-artifact-upload@v2.0.0
       if: ${{ inputs.publish != 'true' }}
       with:
         image: ${{ steps.single_tag.outputs.SINGLE_TAG }}
+        retention_days: "2"
 
     # Code for building multi-platform images and pushing to Docker Hub.
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v3
-      if: ${{ inputs.publish == 'true' }}
+      if: ${{ inputs.publish == 'true' && inputs.depot-project == '' }}
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
-      if: ${{ inputs.publish == 'true' }}
+      if: ${{ inputs.publish == 'true' && inputs.depot-project == '' }}
+    - name: Setup Depot CLI
+      uses: depot/setup-action@v1
+      if: ${{ inputs.publish == 'true' && inputs.depot-project != '' }}
     - name: Login to DockerHub
       uses: docker/login-action@v3
       if: ${{ inputs.publish == 'true' }}
       with:
         username: ${{ inputs.username }}
         password: ${{ inputs.password }}
+
+    # Depot variant.
     - name: Build and Push Multi-Platform image
-      uses: docker/build-push-action@v5
-      if: ${{ inputs.publish == 'true' }}
+      uses: depot/build-push-action@v1
+      if: ${{ inputs.publish == 'true' && inputs.depot-project != '' }}
+      with:
+        project: ${{ inputs.depot-project }}
+        context: ${{ inputs.context }}
+        file: ${{ inputs.file }}
+        platforms: ${{ inputs.platforms }}
+        build-args: ${{ inputs.build-args }}
+        tags: ${{ steps.docker_meta.outputs.tags }}
+        target: ${{ inputs.target }}
+        push: true
+        cache-from: |
+          type=registry,ref=${{ steps.single_tag.outputs.SINGLE_IMAGE }}:head${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }}
+          type=registry,ref=${{ steps.docker_meta.outputs.tags }}
+        cache-to: |
+          type=inline
+
+    - name: Build and Push Multi-Platform image
+      uses: docker/build-push-action@v6
+      if: ${{ inputs.publish == 'true' && inputs.depot-project == '' }}
       with:
         context: ${{ inputs.context }}
         file: ${{ inputs.file }}
@@ -109,7 +150,10 @@ runs:
         tags: ${{ steps.docker_meta.outputs.tags }}
         target: ${{ inputs.target }}
         push: true
-        cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
-        cache-to: type=inline
+        cache-from: |
+          type=registry,ref=${{ steps.single_tag.outputs.SINGLE_IMAGE }}:head${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }}
+          type=registry,ref=${{ steps.docker_meta.outputs.tags }}
+        cache-to: |
+          type=inline
 
     # TODO add code for vuln scanning?

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/pr-labeler-config.yml

@@ -1,22 +1,32 @@
 ingestion:
-  - "metadata-ingestion/**/*"
-  - "metadata-ingestion-modules/**/*"
-  - "metadata-integration/**/*"
+- changed-files:
+  - any-glob-to-any-file:
+    - "metadata-ingestion/**/*"
+    - "metadata-ingestion-modules/**/*"
+    - "metadata-integration/**/*"
 
 devops:
-  - "docker/**/*"
-  - ".github/**/*"
-  - "perf-test/**/*"
-  - "metadata-service/**/*"
+- changed-files:
+  - any-glob-to-any-file:
+    - "docker/**/*"
+    - ".github/**/*"
+    - "perf-test/**/*"
+    - "metadata-service/**/*"
 
 product:
-  - "datahub-web-react/**/*"
-  - "datahub-frontend/**/*"
-  - "datahub-graphql-core/**/*"
-  - "metadata-io/**/*"
+- changed-files:
+  - any-glob-to-any-file:
+    - "datahub-web-react/**/*"
+    - "datahub-frontend/**/*"
+    - "datahub-graphql-core/**/*"
+    - "metadata-io/**/*"
 
 docs:
-  - "docs/**/*"
+- changed-files:
+  - any-glob-to-any-file:
+    - "docs/**/*"
 
 smoke_test:
-  - "smoke-test/**/*"
+- changed-files:
+  - any-glob-to-any-file:
+    - "smoke-test/**/*"

.github/scripts/docker_helpers.sh

@@ -5,22 +5,22 @@ export MAIN_BRANCH="master"
 export MAIN_BRANCH_TAG="head"
 
 function get_short_sha {
-    echo $(git rev-parse --short "$GITHUB_SHA")
+    echo $(git rev-parse --short "$GITHUB_SHA"|head -c7)
 }
 
 export SHORT_SHA=$(get_short_sha)
 echo "SHORT_SHA: $SHORT_SHA"
 
 function get_tag {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g'),${SHORT_SHA}
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
 }
 
 function get_tag_slim {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
 }
 
 function get_tag_full {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
 }
 
 function get_python_docker_release_v {
@@ -38,3 +38,11 @@ function get_unique_tag_slim {
 function get_unique_tag_full {
     echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
 }
+
+function get_platforms_based_on_branch {
+    if [ "${{ github.event_name }}" == 'push' && "${{ github.ref }}" == "refs/heads/${MAIN_BRANCH}" ]; then
+        echo "linux/amd64,linux/arm64"
+    else
+        echo "linux/amd64"
+    fi
+}