Merge branch 'master' into feature/disable-access-mgmt-tab-when-no-roles

Author: githendrik

.github/scripts/check_python_package.py

@@ -0,0 +1,18 @@
+import setuptools
+
+folders = ["./smoke-test/tests"]
+
+for folder in folders:
+    print(f"Checking folder {folder}")
+    a = [i for i in setuptools.find_packages(folder) if "cypress" not in i]
+    b = [i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i]
+
+    in_a_not_b = set(a) - set(b)
+    in_b_not_a = set(b) - set(a)
+
+    assert (
+        len(in_a_not_b) == 0
+    ), f"Found packages in {folder} that are not in namespace packages: {in_a_not_b}"
+    assert (
+        len(in_b_not_a) == 0
+    ), f"Found namespace packages in {folder} that are not in packages: {in_b_not_a}"

.github/workflows/airflow-plugin.yml

@@ -87,8 +87,8 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           directory: .
           fail_ci_if_error: false
-          flags: airflow-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
-          name: pytest-airflow
+          flags: airflow,airflow-${{ matrix.extra_pip_extras }}
+          name: pytest-airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_requirements }}
           verbose: true
 
   event-file:

.github/workflows/docker-unified.yml

@@ -37,8 +37,29 @@ env:
   DATAHUB_INGESTION_IMAGE: "acryldata/datahub-ingestion"
 
 jobs:
+  check_lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: hsheth2/sane-checkout-action@v1
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: 17
+      - name: Ensure packages are correct
+        run: |
+          python ./.github/scripts/check_python_package.py
+      - name: Run lint on smoke test
+        run: |
+          ./gradlew :smoke-test:lint
   setup:
     runs-on: ubuntu-latest
+    needs: check_lint
     outputs:
       tag: ${{ steps.tag.outputs.tag }}
       slim_tag: ${{ steps.tag.outputs.slim_tag }}

.github/workflows/metadata-ingestion.yml

@@ -31,7 +31,7 @@ jobs:
       # DATAHUB_LOOKML_GIT_TEST_SSH_KEY: ${{ secrets.DATAHUB_LOOKML_GIT_TEST_SSH_KEY }}
     strategy:
       matrix:
-        python-version: ["3.7", "3.10"]
+        python-version: ["3.8", "3.10"]
         command:
           [
             "testQuick",
@@ -40,7 +40,7 @@ jobs:
             "testIntegrationBatch2",
           ]
         include:
-          - python-version: "3.7"
+          - python-version: "3.8"
           - python-version: "3.10"
       fail-fast: false
     steps:

.github/workflows/spark-smoke-test.yml

@@ -42,8 +42,12 @@ jobs:
           cache: "pip"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
+      - name: Disk Check
+        run: df -h . && docker images
       - name: Remove images
         run: docker image prune -a -f || true
+      - name: Disk Check
+        run: df -h . && docker images
       - name: Smoke test
         run: |
           ./gradlew :metadata-integration:java:spark-lineage:integrationTest \
@@ -54,12 +58,24 @@ jobs:
              -x :datahub-web-react:yarnBuild \
              -x :datahub-web-react:distZip \
              -x :datahub-web-react:jar
+      - name: store logs
+        if: failure()
+        run: |
+          docker ps -a
+          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log || true
+          docker logs broker >& broker-${{ matrix.test_strategy }}.log || true
+          docker logs mysql >& mysql-${{ matrix.test_strategy }}.log || true
+          docker logs elasticsearch >& elasticsearch-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-frontend-react >& frontend-${{ matrix.test_strategy }}.log || true
       - name: Upload logs
         uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: docker logs
-          path: "docker/build/container-logs/*.log"
+          path: |
+            "**/build/container-logs/*.log"
+            "*.log"
       - uses: actions/upload-artifact@v3
         if: always()
         with:

build.gradle

@@ -30,7 +30,7 @@ buildscript {
 
   ext.junitJupiterVersion = '5.6.1'
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
-  ext.pegasusVersion = '29.48.4'
+  ext.pegasusVersion = '29.51.0'
   ext.mavenVersion = '3.6.3'
   ext.springVersion = '6.1.2'
   ext.springBootVersion = '3.2.1'
@@ -221,7 +221,7 @@ project.ext.externalDependency = [
     'resilience4j': 'io.github.resilience4j:resilience4j-retry:1.7.1',
     'rythmEngine': 'org.rythmengine:rythm-engine:1.3.0',
     'servletApi': 'jakarta.servlet:jakarta.servlet-api:6.0.0',
-    'shiroCore': 'org.apache.shiro:shiro-core:1.11.0',
+    'shiroCore': 'org.apache.shiro:shiro-core:1.13.0',
     'snakeYaml': 'org.yaml:snakeyaml:2.0',
     'sparkSql' : 'org.apache.spark:spark-sql_2.12:3.0.3',
     'sparkHive' : 'org.apache.spark:spark-hive_2.12:3.0.3',
@@ -269,13 +269,14 @@ allprojects {
   apply plugin: 'eclipse'
   // apply plugin: 'org.gradlex.java-ecosystem-capabilities'
 
-  tasks.withType(Test).configureEach {
-    // https://docs.gradle.org/current/userguide/performance.html
-    maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+  tasks.withType(Test).configureEach { task -> if (task.project.name != "metadata-io") {
+      // https://docs.gradle.org/current/userguide/performance.html
+      maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
 
-    if (project.configurations.getByName("testImplementation").getDependencies()
-            .any{ it.getName().contains("testng") }) {
-      useTestNG()
+      if (project.configurations.getByName("testImplementation").getDependencies()
+              .any { it.getName().contains("testng") }) {
+        useTestNG()
+      }
     }
   }
 

buildSrc/build.gradle

@@ -25,7 +25,4 @@ dependencies {
 
   compileOnly 'org.projectlombok:lombok:1.18.30'
   annotationProcessor 'org.projectlombok:lombok:1.18.30'
-
-  // pegasus dependency, overrides for tasks
-  implementation 'com.linkedin.pegasus:gradle-plugins:29.48.4'
 }