Merge branch 'master' into prefect-source-integration

Author: treff7es

.github/scripts/check_python_package.py

@@ -0,0 +1,18 @@
+import setuptools
+
+folders = ["./smoke-test/tests"]
+
+for folder in folders:
+    print(f"Checking folder {folder}")
+    a = [i for i in setuptools.find_packages(folder) if "cypress" not in i]
+    b = [i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i]
+
+    in_a_not_b = set(a) - set(b)
+    in_b_not_a = set(b) - set(a)
+
+    assert (
+        len(in_a_not_b) == 0
+    ), f"Found packages in {folder} that are not in namespace packages: {in_a_not_b}"
+    assert (
+        len(in_b_not_a) == 0
+    ), f"Found namespace packages in {folder} that are not in packages: {in_b_not_a}"

.github/workflows/docker-unified.yml

@@ -37,8 +37,29 @@ env:
   DATAHUB_INGESTION_IMAGE: "acryldata/datahub-ingestion"
 
 jobs:
+  check_lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: hsheth2/sane-checkout-action@v1
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: 17
+      - name: Ensure packages are correct
+        run: |
+          python ./.github/scripts/check_python_package.py
+      - name: Run lint on smoke test
+        run: |
+          ./gradlew :smoke-test:lint
   setup:
     runs-on: ubuntu-latest
+    needs: check_lint
     outputs:
       tag: ${{ steps.tag.outputs.tag }}
       slim_tag: ${{ steps.tag.outputs.slim_tag }}

build.gradle

@@ -30,7 +30,7 @@ buildscript {
 
   ext.junitJupiterVersion = '5.6.1'
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
-  ext.pegasusVersion = '29.48.4'
+  ext.pegasusVersion = '29.51.0'
   ext.mavenVersion = '3.6.3'
   ext.springVersion = '6.1.2'
   ext.springBootVersion = '3.2.1'
@@ -269,13 +269,14 @@ allprojects {
   apply plugin: 'eclipse'
   // apply plugin: 'org.gradlex.java-ecosystem-capabilities'
 
-  tasks.withType(Test).configureEach {
-    // https://docs.gradle.org/current/userguide/performance.html
-    maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+  tasks.withType(Test).configureEach { task -> if (task.project.name != "metadata-io") {
+      // https://docs.gradle.org/current/userguide/performance.html
+      maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
 
-    if (project.configurations.getByName("testImplementation").getDependencies()
-            .any{ it.getName().contains("testng") }) {
-      useTestNG()
+      if (project.configurations.getByName("testImplementation").getDependencies()
+              .any { it.getName().contains("testng") }) {
+        useTestNG()
+      }
     }
   }
 

buildSrc/build.gradle

@@ -25,7 +25,4 @@ dependencies {
 
   compileOnly 'org.projectlombok:lombok:1.18.30'
   annotationProcessor 'org.projectlombok:lombok:1.18.30'
-
-  // pegasus dependency, overrides for tasks
-  implementation 'com.linkedin.pegasus:gradle-plugins:29.48.4'
 }