feat(python): split out temp wheel builds (datahub-project#12157)

hsheth2 · yoonhyejin · commit d2974245ed59 · 2024-12-23T15:32:42.000+09:00
diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -27,7 +27,6 @@ jobs:
   airflow-plugin:
     runs-on: ubuntu-latest
     env:
-      SPARK_VERSION: 3.0.3
       DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
@@ -69,7 +68,7 @@ jobs:
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: ${{ always() && matrix.python-version == '3.10' && matrix.extra_pip_requirements == 'apache-airflow>=2.7.0' }}
         with:
           name: Test Results (Airflow Plugin ${{ matrix.python-version}})
@@ -93,7 +92,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Event File
           path: ${{ github.event_path }}
diff --git a/.github/workflows/dagster-plugin.yml b/.github/workflows/dagster-plugin.yml
@@ -27,7 +27,6 @@ jobs:
   dagster-plugin:
     runs-on: ubuntu-latest
     env:
-      SPARK_VERSION: 3.0.3
       DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
@@ -44,7 +43,8 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: actions/checkout@v4
+      - uses: gradle/actions/setup-gradle@v3
+      - uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
@@ -56,7 +56,7 @@ jobs:
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && uv pip freeze
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }}
         with:
           name: Test Results (dagster Plugin ${{ matrix.python-version}})
@@ -79,7 +79,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Event File
           path: ${{ github.event_path }}
diff --git a/.github/workflows/gx-plugin.yml b/.github/workflows/gx-plugin.yml
@@ -27,7 +27,6 @@ jobs:
   gx-plugin:
     runs-on: ubuntu-latest
     env:
-      SPARK_VERSION: 3.0.3
       DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
@@ -48,7 +47,8 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: actions/checkout@v4
+      - uses: gradle/actions/setup-gradle@v3
+      - uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
@@ -60,7 +60,7 @@ jobs:
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && uv pip freeze
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'great-expectations~=0.17.0' }}
         with:
           name: Test Results (GX Plugin ${{ matrix.python-version}})
@@ -83,7 +83,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Event File
           path: ${{ github.event_path }}
diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -28,7 +28,6 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 40
     env:
-      SPARK_VERSION: 3.3.2
       DATAHUB_TELEMETRY_ENABLED: false
       # TODO: Enable this once the test is fixed.
       # DATAHUB_LOOKML_GIT_TEST_SSH_KEY: ${{ secrets.DATAHUB_LOOKML_GIT_TEST_SSH_KEY }}
@@ -84,9 +83,9 @@ jobs:
           df -hl
           docker image ls
           docker system df
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
-          name: Test Results (metadata ingestion ${{ matrix.python-version }})
+          name: Test Results (metadata ingestion ${{ matrix.python-version }} ${{ matrix.command }})
           path: |
             **/build/reports/tests/test/**
             **/build/test-results/test/**
@@ -100,14 +99,14 @@ jobs:
           directory: ./build/coverage-reports/
           fail_ci_if_error: false
           flags: pytest-${{ matrix.command }}
-          name: pytest-${{ matrix.command }}
+          name: pytest-${{ matrix.python-version }}-${{ matrix.command }}
           verbose: true
 
   event-file:
     runs-on: ubuntu-latest
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Event File
           path: ${{ github.event_path }}
diff --git a/.github/workflows/prefect-plugin.yml b/.github/workflows/prefect-plugin.yml
@@ -27,25 +27,20 @@ jobs:
   prefect-plugin:
     runs-on: ubuntu-latest
     env:
-      SPARK_VERSION: 3.0.3
       DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
         python-version: ["3.8", "3.9", "3.10"]
-        include:
-          - python-version: "3.8"
-          - python-version: "3.9"
-          - python-version: "3.10"
       fail-fast: false
     steps:
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: acryldata/sane-checkout-action@v3
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
@@ -56,7 +51,7 @@ jobs:
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && uv pip freeze
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: ${{ always() && matrix.python-version == '3.10'}}
         with:
           name: Test Results (Prefect Plugin ${{ matrix.python-version}})
@@ -72,15 +67,15 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           directory: ./build/coverage-reports/
           fail_ci_if_error: false
-          flags: prefect,prefect-${{ matrix.extra_pip_extras }}
+          flags: prefect,prefect-${{ matrix.python-version }}
           name: pytest-prefect-${{ matrix.python-version }}
           verbose: true
 
   event-file:
     runs-on: ubuntu-latest
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Event File
           path: ${{ github.event_path }}
diff --git a/.github/workflows/python-build-pages.yml b/.github/workflows/python-build-pages.yml
@@ -0,0 +1,64 @@
+name: Python Build
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - ".github/workflows/python-build-pages.yml"
+      - "metadata-ingestion/**"
+      - "metadata-ingestion-modules/**"
+      - "metadata-models/**"
+  pull_request:
+    branches:
+      - "**"
+    paths:
+      - ".github/workflows/python-build-pages.yml"
+      - "metadata-ingestion/**"
+      - "metadata-ingestion-modules/**"
+      - "metadata-models/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  deploy-pages:
+    runs-on: ubuntu-latest
+    if: ${{ vars.CLOUDFLARE_WHEELS_PROJECT_NAME != '' }}
+
+    name: Python Wheels
+    permissions:
+      contents: read
+      pull-requests: read
+      deployments: write
+    steps:
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: "zulu"
+          java-version: 17
+      - uses: gradle/actions/setup-gradle@v3
+      - uses: acryldata/sane-checkout-action@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "pip"
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
+      - name: Build Python wheel site
+        run: |
+          ./gradlew :python-build:buildSite
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - name: Publish
+        uses: cloudflare/pages-action@v1
+        with:
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          projectName: ${{ vars.CLOUDFLARE_WHEELS_PROJECT_NAME }}
+          workingDirectory: python-build
+          directory: site
+          gitHubToken: ${{ github.token }}
diff --git a/docs-website/build.gradle b/docs-website/build.gradle
@@ -83,11 +83,7 @@ task yarnInstall(type: YarnTask) {
 task yarnGenerate(type: YarnTask, dependsOn: [yarnInstall,
     generateGraphQLSchema, generateJsonSchema,
     ':metadata-ingestion:modelDocGen', ':metadata-ingestion:docGen',
-    ':metadata-ingestion:buildWheel',
-    ':metadata-ingestion-modules:airflow-plugin:buildWheel',
-    ':metadata-ingestion-modules:dagster-plugin:buildWheel',
-    ':metadata-ingestion-modules:prefect-plugin:buildWheel',
-    ':metadata-ingestion-modules:gx-plugin:buildWheel',
+    ':python-build:buildWheels',
     ]) {
   inputs.files(projectMdFiles)
   outputs.cacheIf { true }
diff --git a/docs-website/generateDocsDir.ts b/docs-website/generateDocsDir.ts
@@ -573,26 +573,20 @@ function write_markdown_file(
 
 function copy_python_wheels(): void {
   // Copy the built wheel files to the static directory.
-  const wheel_dirs = [
-    "../metadata-ingestion/dist",
-    "../metadata-ingestion-modules/airflow-plugin/dist",
-    "../metadata-ingestion-modules/dagster-plugin/dist",
-    "../metadata-ingestion-modules/prefect-plugin/dist",
-    "../metadata-ingestion-modules/gx-plugin/dist",
-  ];
+  // Everything is copied to the python-build directory first, so
+  // we just need to copy from there.
+  const wheel_dir = "../python-build/wheels";
 
   const wheel_output_directory = path.join(STATIC_DIRECTORY, "wheels");
   fs.mkdirSync(wheel_output_directory, { recursive: true });
 
-  for (const wheel_dir of wheel_dirs) {
-    const wheel_files = fs.readdirSync(wheel_dir);
-    for (const wheel_file of wheel_files) {
-      const src = path.join(wheel_dir, wheel_file);
-      const dest = path.join(wheel_output_directory, wheel_file);
+  const wheel_files = fs.readdirSync(wheel_dir);
+  for (const wheel_file of wheel_files) {
+    const src = path.join(wheel_dir, wheel_file);
+    const dest = path.join(wheel_output_directory, wheel_file);
 
-      // console.log(`Copying artifact ${src} to ${dest}...`);
-      fs.copyFileSync(src, dest);
-    }
+    // console.log(`Copying artifact ${src} to ${dest}...`);
+    fs.copyFileSync(src, dest);
   }
 }
 
diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle
@@ -23,8 +23,8 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
   inputs.file file('setup.py')
   outputs.file(sentinel_file)
   commandLine 'bash', '-c',
-    "${python_executable} -m venv ${venv_name} && " +
-    "${venv_name}/bin/python -m pip install --upgrade pip uv wheel 'setuptools>=63.0.0' && " +
+    "${python_executable} -m venv ${venv_name} && set -x && " +
+    "${venv_name}/bin/python -m pip install --upgrade uv && " +
     "touch ${sentinel_file}"
 }
 
diff --git a/python-build/.gitignore b/python-build/.gitignore
@@ -0,0 +1,3 @@
+
+/wheels
+/site
diff --git a/python-build/build.gradle b/python-build/build.gradle
@@ -0,0 +1,27 @@
+plugins {
+  id 'base'
+}
+
+ext {
+  python_executable = 'python3'
+}
+
+task checkPythonVersion(type: Exec) {
+  commandLine python_executable, '-c',
+    'import sys; sys.version_info >= (3, 8), f"Python version {sys.version_info} is too old"'
+}
+
+task buildWheels(type: Exec, dependsOn: [
+  checkPythonVersion,
+  ':metadata-ingestion:buildWheel',
+  ':metadata-ingestion-modules:airflow-plugin:buildWheel',
+  ':metadata-ingestion-modules:dagster-plugin:buildWheel',
+  ':metadata-ingestion-modules:prefect-plugin:buildWheel',
+  ':metadata-ingestion-modules:gx-plugin:buildWheel',
+]) {
+  commandLine python_executable, "copy_wheels.py"
+}
+
+task buildSite(type: Exec, dependsOn: [buildWheels]) {
+  commandLine python_executable, "build_site.py"
+}
diff --git a/python-build/build_site.py b/python-build/build_site.py
diff --git a/python-build/copy_wheels.py b/python-build/copy_wheels.py
diff --git a/settings.gradle b/settings.gradle
