Merge branch 'master' of https://github.com/shubhamjagtap639/datahub into prefect-source-integration

Author: shubhamjagtap639

.dockerignore

@@ -1,6 +1,7 @@
 **/node_modules/
-datahub-frontend/build/
-metadata-ingestion/venv/
+*/build/
+*/*/build/
+*/venv/
 out
 **/*.class
 # Have to copy gradle/wrapper/gradle-wrapper.jar, can't exclude ALL jars

.github/actions/ci-optimization/action.yml

@@ -0,0 +1,79 @@
+name: 'Identify CI Optimizations'
+description: 'Determine if code changes are specific to certain modules.'
+
+outputs:
+  frontend-only:
+    description: "Frontend only change"
+    value: ${{ steps.filter.outputs.frontend == 'true' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'false' }}
+  ingestion-only:
+    description: "Ingestion only change"
+    value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'true' && steps.filter.outputs.backend == 'false' }}
+  backend-only:
+    description: "Backend only change"
+    value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'true' }}
+  backend-change:
+    description: "Backend code has changed"
+    value: ${{ steps.filter.outputs.backend == 'true' }}
+  ingestion-change:
+    description: "Ingestion code has changed"
+    value: ${{ steps.filter.outputs.ingestion == 'true' }}
+  frontend-change:
+    description: "Frontend code has changed"
+    value: ${{ steps.filter.outputs.frontend == 'true' }}
+  docker-change:
+    description: "Docker code has changed"
+    value: ${{ steps.filter.outputs.docker == 'true' }}
+  kafka-setup-change:
+    description: "Kafka setup docker change"
+    value: ${{ steps.filter.outputs.kafka-setup == 'true' }}
+  mysql-setup-change:
+    description: "Mysql setup docker change"
+    value: ${{ steps.filter.outputs.mysql-setup == 'true' }}
+  postgres-setup-change:
+    description: "Postgres setup docker change"
+    value: ${{ steps.filter.outputs.postgres-setup == 'true' }}
+  elasticsearch-setup-change:
+    description: "Elasticsearch setup docker change"
+    value: ${{ steps.filter.outputs.elasticsearch-setup == 'true' }}
+runs:
+  using: "composite"
+  steps:
+    - uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        filters: |
+          frontend:
+            - "datahub-frontend/**"
+            - "datahub-web-react/**"
+            - "smoke-test/tests/cypress/**"
+            - "docker/datahub-frontend/**"
+          ingestion:
+            - "metadata-ingestion-modules/**"
+            - "metadata-ingestion/**"
+            - "metadata-models/**"
+            - "smoke-test/**"
+            - "docker/datahub-ingestion-**"
+          docker:
+            - "docker/**"
+          backend:
+            - "metadata-models/**"
+            - "datahub-upgrade/**"
+            - "entity-registry/**"
+            - "li-utils/**"
+            - "metadata-auth/**"
+            - "metadata-dao-impl/**"
+            - "metadata-events/**"
+            - "metadata-io/**"
+            - "metadata-jobs/**"
+            - "metadata-service/**"
+            - "metadata-utils/**"
+            - "smoke-test/**"
+            - "docker/**"
+          kafka-setup:
+            - "docker/kafka-setup/**"
+          mysql-setup:
+            - "docker/mysql-setup/**"
+          postgres-setup:
+            - "docker/postgres-setup/**"
+          elasticsearch-setup:
+            - "docker/elasticsearch-setup/**"

.github/actions/docker-custom-build-and-push/action.yml

@@ -70,11 +70,20 @@ runs:
         push: false
         cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
         cache-to: type=inline
+    - name: Single Tag
+      if: ${{ inputs.publish != 'true' }}
+      shell: bash
+      run: |
+        TAGS="""
+        ${{ steps.docker_meta.outputs.tags }}
+        """
+        echo "SINGLE_TAG=$(echo $TAGS | tr '\n' ' ' | awk -F' ' '{ print $1 }')" >> $GITHUB_OUTPUT
+      id: single_tag
     - name: Upload image locally for testing (if not publishing)
       uses: ishworkh/docker-image-artifact-upload@v1
       if: ${{ inputs.publish != 'true' }}
       with:
-        image: ${{ steps.docker_meta.outputs.tags }}
+        image: ${{ steps.single_tag.outputs.SINGLE_TAG }}
 
     # Code for building multi-platform images and pushing to Docker Hub.
     - name: Set up QEMU

.github/scripts/check_policies.py

@@ -20,7 +20,7 @@
     elif urn == "urn:li:dataHubPolicy:editor-platform-policy":
         editor_platform_policy_privileges = policy["info"]["privileges"]
     elif urn == "urn:li:dataHubPolicy:7":
-        all_user_platform_policy_privilges = policy["info"]["privileges"]
+        all_user_platform_policy_privileges = policy["info"]["privileges"]
     try:
         doc_type = policy["info"]["type"]
         privileges = policy["info"]["privileges"]
@@ -54,10 +54,22 @@
 )
 assert len(diff_policies) == 0, f"Missing privileges for root user are {diff_policies}"
 
-diff_policies = set(editor_platform_policy_privileges).difference(
-    set(all_user_platform_policy_privilges)
-)
-assert "MANAGE_POLICIES" not in all_user_platform_policy_privilges
-assert (
-    len(diff_policies) == 0
-), f"Missing privileges for all user policies are {diff_policies}"
+# All users privileges checks
+assert "MANAGE_POLICIES" not in all_user_platform_policy_privileges
+assert "MANAGE_USERS_AND_GROUPS" not in all_user_platform_policy_privileges
+assert "MANAGE_SECRETS" not in all_user_platform_policy_privileges
+assert "MANAGE_USER_CREDENTIALS" not in all_user_platform_policy_privileges
+assert "MANAGE_ACCESS_TOKENS" not in all_user_platform_policy_privileges
+assert "EDIT_ENTITY" not in all_user_platform_policy_privileges
+assert "DELETE_ENTITY" not in all_user_platform_policy_privileges
+
+# Editor checks
+assert "MANAGE_POLICIES" not in editor_platform_policy_privileges
+assert "MANAGE_USERS_AND_GROUPS" not in editor_platform_policy_privileges
+assert "MANAGE_SECRETS" not in editor_platform_policy_privileges
+assert "MANAGE_USER_CREDENTIALS" not in editor_platform_policy_privileges
+assert "MANAGE_ACCESS_TOKENS" not in editor_platform_policy_privileges
+# These don't prevent a user from modifying entities they are an asset owner of, i.e. their own profile info
+assert "EDIT_CONTACT_INFO" not in editor_platform_policy_privileges
+assert "EDIT_USER_PROFILE" not in editor_platform_policy_privileges
+assert "EDIT_ENTITY_OWNERS" not in editor_platform_policy_privileges

.github/scripts/docker_helpers.sh

@@ -12,7 +12,15 @@ export SHORT_SHA=$(get_short_sha)
 echo "SHORT_SHA: $SHORT_SHA"
 
 function get_tag {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g'),${SHORT_SHA}
+}
+
+function get_tag_slim {
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim
+}
+
+function get_tag_full {
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full
 }
 
 function get_python_docker_release_v {
@@ -21,4 +29,12 @@ function get_python_docker_release_v {
 
 function get_unique_tag {
     echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+}
+
+function get_unique_tag_slim {
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
+}
+
+function get_unique_tag_full {
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
 }

.github/workflows/airflow-plugin.yml

@@ -10,9 +10,9 @@ on:
       - "metadata-models/**"
   pull_request:
     branches:
-      - master
+      - "**"
     paths:
-      - ".github/**"
+      - ".github/workflows/airflow-plugin.yml"
       - "metadata-ingestion-modules/airflow-plugin/**"
       - "metadata-ingestion/**"
       - "metadata-models/**"
@@ -32,47 +32,63 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: "3.7"
-            extraPythonRequirement: "apache-airflow~=2.1.0"
-          - python-version: "3.7"
-            extraPythonRequirement: "apache-airflow~=2.2.0"
+          # Note: this should be kept in sync with tox.ini.
+          - python-version: "3.8"
+            extra_pip_requirements: "apache-airflow~=2.1.4"
+            extra_pip_extras: plugin-v1
+          - python-version: "3.8"
+            extra_pip_requirements: "apache-airflow~=2.2.4"
+            extra_pip_extras: plugin-v1
           - python-version: "3.10"
-            extraPythonRequirement: "apache-airflow~=2.4.0"
+            extra_pip_requirements: 'apache-airflow~=2.4.0 pluggy==1.0.0 "pendulum<3.0" "Flask-Session<0.6.0"'
+            extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extraPythonRequirement: "apache-airflow~=2.6.0"
+            extra_pip_requirements: 'apache-airflow~=2.6.0 "pendulum<3.0" "Flask-Session<0.6.0"'
+            extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extraPythonRequirement: "apache-airflow>2.6.0"
+            extra_pip_requirements: 'apache-airflow~=2.7.0 pydantic==2.4.2 "Flask-Session<0.6.0"'
+            extra_pip_extras: plugin-v2
+          - python-version: "3.10"
+            extra_pip_requirements: 'apache-airflow>=2.8.0 pydantic>=2.4.2 "Flask-Session<0.6.0"'
+            extra_pip_extras: plugin-v2
       fail-fast: false
     steps:
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
-      - name: Install airflow package and test  (extras ${{ matrix.extraPythonRequirement }})
-        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:airflow-plugin:lint :metadata-ingestion-modules:airflow-plugin:testQuick
+      - name: Install airflow package and test (extras ${{ matrix.extra_pip_requirements }})
+        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:lint :metadata-ingestion-modules:airflow-plugin:testQuick
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze
       - uses: actions/upload-artifact@v3
-        if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'apache-airflow>2.6.0' }}
+        if: ${{ always() && matrix.python-version == '3.10' && matrix.extra_pip_requirements == 'apache-airflow>=2.7.0' }}
         with:
           name: Test Results (Airflow Plugin ${{ matrix.python-version}})
           path: |
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
+            !**/binary/**
       - name: Upload coverage to Codecov
         if: always()
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           directory: .
           fail_ci_if_error: false
-          flags: airflow-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
-          name: pytest-airflow
+          flags: airflow,airflow-${{ matrix.extra_pip_extras }}
+          name: pytest-airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_requirements }}
           verbose: true
 
   event-file: