Skip to content

Commit 0cbb48c

Browse files
kanavnarulakanavnarula
authored
Merge branch 'master' into container-hierarchy-changes
2 parents 2d642c0 + 70bec48 commit 0cbb48c

File tree

960 files changed

+100428
-9695
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

960 files changed

+100428
-9695
lines changed

‎.github/workflows/build-and-test.yml

+4
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@ jobs:
7575
path: |
7676
~/.cache/uv
7777
key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
78+
- name: Install dependencies
79+
run: ./metadata-ingestion/scripts/install_deps.sh
7880
- name: Set up JDK 17
7981
uses: actions/setup-java@v4
8082
with:
@@ -83,6 +85,7 @@ jobs:
8385
- uses: gradle/actions/setup-gradle@v3
8486
- name: Gradle build (and test) for NOT metadata ingestion
8587
if: ${{ matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
88+
# datahub-schematron:cli excluded due to dependency on metadata-ingestion
8689
run: |
8790
./gradlew build \
8891
-x :metadata-ingestion:build \
@@ -100,6 +103,7 @@ jobs:
100103
-x :metadata-ingestion-modules:gx-plugin:check \
101104
-x :datahub-frontend:build \
102105
-x :datahub-web-react:build \
106+
-x :metadata-integration:java:datahub-schematron:cli:test \
103107
--parallel
104108
- name: Gradle build (and test) for frontend
105109
if: ${{ matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }}

‎.github/workflows/check-datahub-jars.yml

+1
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,5 @@ jobs:
4040
- name: check ${{ matrix.command }} jar
4141
run: |
4242
./gradlew :metadata-integration:java:${{ matrix.command }}:build --info
43+
./gradlew :metadata-integration:java:${{ matrix.command }}:checkShadowJar
4344
./gradlew :metadata-integration:java:${{ matrix.command }}:javadoc

‎.github/workflows/docker-unified.yml

+65
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,11 @@ jobs:
144144
needs: setup
145145
if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
146146
steps:
147+
- name: Free up disk space
148+
run: |
149+
sudo apt-get remove 'dotnet-*' azure-cli || true
150+
sudo rm -rf /usr/local/lib/android/ || true
151+
sudo docker image prune -a -f || true
147152
- name: Set up JDK 17
148153
uses: actions/setup-java@v4
149154
with:
@@ -210,6 +215,11 @@ jobs:
210215
needs: setup
211216
if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
212217
steps:
218+
- name: Free up disk space
219+
run: |
220+
sudo apt-get remove 'dotnet-*' azure-cli || true
221+
sudo rm -rf /usr/local/lib/android/ || true
222+
sudo docker image prune -a -f || true
213223
- name: Set up JDK 17
214224
uses: actions/setup-java@v4
215225
with:
@@ -276,6 +286,11 @@ jobs:
276286
needs: setup
277287
if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
278288
steps:
289+
- name: Free up disk space
290+
run: |
291+
sudo apt-get remove 'dotnet-*' azure-cli || true
292+
sudo rm -rf /usr/local/lib/android/ || true
293+
sudo docker image prune -a -f || true
279294
- name: Set up JDK 17
280295
uses: actions/setup-java@v4
281296
with:
@@ -342,6 +357,11 @@ jobs:
342357
needs: setup
343358
if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
344359
steps:
360+
- name: Free up disk space
361+
run: |
362+
sudo apt-get remove 'dotnet-*' azure-cli || true
363+
sudo rm -rf /usr/local/lib/android/ || true
364+
sudo docker image prune -a -f || true
345365
- name: Set up JDK 17
346366
uses: actions/setup-java@v4
347367
with:
@@ -408,6 +428,11 @@ jobs:
408428
needs: setup
409429
if: ${{ needs.setup.outputs.frontend_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true'}}
410430
steps:
431+
- name: Free up disk space
432+
run: |
433+
sudo apt-get remove 'dotnet-*' azure-cli || true
434+
sudo rm -rf /usr/local/lib/android/ || true
435+
sudo docker image prune -a -f || true
411436
- name: Set up JDK 17
412437
uses: actions/setup-java@v4
413438
with:
@@ -476,6 +501,11 @@ jobs:
476501
needs: setup
477502
if: ${{ needs.setup.outputs.kafka_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true') }}
478503
steps:
504+
- name: Free up disk space
505+
run: |
506+
sudo apt-get remove 'dotnet-*' azure-cli || true
507+
sudo rm -rf /usr/local/lib/android/ || true
508+
sudo docker image prune -a -f || true
479509
- name: Check out the repo
480510
uses: acryldata/sane-checkout-action@v3
481511
- name: Build and push
@@ -532,6 +562,11 @@ jobs:
532562
needs: setup
533563
if: ${{ needs.setup.outputs.mysql_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true') }}
534564
steps:
565+
- name: Free up disk space
566+
run: |
567+
sudo apt-get remove 'dotnet-*' azure-cli || true
568+
sudo rm -rf /usr/local/lib/android/ || true
569+
sudo docker image prune -a -f || true
535570
- name: Check out the repo
536571
uses: acryldata/sane-checkout-action@v3
537572
- name: Build and push
@@ -588,6 +623,11 @@ jobs:
588623
needs: setup
589624
if: ${{ needs.setup.outputs.elasticsearch_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' ) }}
590625
steps:
626+
- name: Free up disk space
627+
run: |
628+
sudo apt-get remove 'dotnet-*' azure-cli || true
629+
sudo rm -rf /usr/local/lib/android/ || true
630+
sudo docker image prune -a -f || true
591631
- name: Check out the repo
592632
uses: acryldata/sane-checkout-action@v3
593633
- name: Build and push
@@ -646,6 +686,11 @@ jobs:
646686
needs: setup
647687
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
648688
steps:
689+
- name: Free up disk space
690+
run: |
691+
sudo apt-get remove 'dotnet-*' azure-cli || true
692+
sudo rm -rf /usr/local/lib/android/ || true
693+
sudo docker image prune -a -f || true
649694
- name: Check out the repo
650695
uses: acryldata/sane-checkout-action@v3
651696
- name: Build and push Base Image
@@ -674,6 +719,11 @@ jobs:
674719
needs: [setup, datahub_ingestion_base_build]
675720
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
676721
steps:
722+
- name: Free up disk space
723+
run: |
724+
sudo apt-get remove 'dotnet-*' azure-cli || true
725+
sudo rm -rf /usr/local/lib/android/ || true
726+
sudo docker image prune -a -f || true
677727
- name: Check out the repo
678728
uses: acryldata/sane-checkout-action@v3
679729
- name: Download Base Image
@@ -717,6 +767,11 @@ jobs:
717767
needs: [setup, datahub_ingestion_base_build]
718768
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
719769
steps:
770+
- name: Free up disk space
771+
run: |
772+
sudo apt-get remove 'dotnet-*' azure-cli || true
773+
sudo rm -rf /usr/local/lib/android/ || true
774+
sudo docker image prune -a -f || true
720775
- name: Check out the repo
721776
uses: acryldata/sane-checkout-action@v3
722777
- name: Download Base Image
@@ -760,6 +815,11 @@ jobs:
760815
needs: [setup, datahub_ingestion_base_slim_build]
761816
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
762817
steps:
818+
- name: Free up disk space
819+
run: |
820+
sudo apt-get remove 'dotnet-*' azure-cli || true
821+
sudo rm -rf /usr/local/lib/android/ || true
822+
sudo docker image prune -a -f || true
763823
- name: Check out the repo
764824
uses: acryldata/sane-checkout-action@v3
765825
- uses: actions/setup-python@v5
@@ -856,6 +916,11 @@ jobs:
856916
needs: [setup, datahub_ingestion_base_full_build]
857917
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
858918
steps:
919+
- name: Free up disk space
920+
run: |
921+
sudo apt-get remove 'dotnet-*' azure-cli || true
922+
sudo rm -rf /usr/local/lib/android/ || true
923+
sudo docker image prune -a -f || true
859924
- name: Check out the repo
860925
uses: acryldata/sane-checkout-action@v3
861926
- uses: actions/setup-python@v5

‎.github/workflows/gx-plugin.yml

+2
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ jobs:
3939
extraPythonRequirement: "great-expectations~=0.16.0 numpy~=1.26.0"
4040
- python-version: "3.11"
4141
extraPythonRequirement: "great-expectations~=0.17.0"
42+
- python-version: "3.11"
43+
extraPythonRequirement: "great-expectations~=0.18.0"
4244
fail-fast: false
4345
steps:
4446
- name: Set up JDK 17

‎.github/workflows/pr-labeler.yml

+5-4
Original file line numberDiff line numberDiff line change
@@ -29,23 +29,24 @@ jobs:
2929
"swaroopjagadish",
3030
"treff7es",
3131
"yoonhyejin",
32-
"eboneil",
3332
"gabe-lyons",
3433
"hsheth2",
3534
"jjoyce0510",
3635
"maggiehays",
3736
"pedro93",
3837
"RyanHolstien",
3938
"sakethvarma397",
40-
"Kunal-kankriya",
4139
"purnimagarg1",
42-
"dushayntAW",
4340
"sagar-salvi-apptware",
4441
"kushagra-apptware",
4542
"Salman-Apptware",
4643
"mayurinehate",
4744
"noggi",
48-
"skrydal"
45+
"skrydal",
46+
"kevinkarchacryl",
47+
"sgomezvillamor",
48+
"acrylJonny",
49+
"chakru-r"
4950
]'),
5051
github.actor
5152
)

‎build.gradle

+28-15
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,10 @@ buildscript {
4545
ext.elasticsearchVersion = '2.11.1' // ES 7.10, Opensearch 1.x, 2.x
4646
ext.jacksonVersion = '2.15.3'
4747
ext.jettyVersion = '11.0.21'
48+
// see also datahub-frontend/play.gradle
4849
ext.playVersion = '2.8.22'
50+
ext.playScalaVersion = '2.13'
51+
ext.akkaVersion = '2.6.21' // 2.7.0+ has incompatible license
4952
ext.log4jVersion = '2.23.1'
5053
ext.slf4jVersion = '1.7.36'
5154
ext.logbackClassic = '1.4.14'
@@ -54,7 +57,7 @@ buildscript {
5457
ext.hazelcastVersion = '5.3.6'
5558
ext.ebeanVersion = '15.5.2'
5659
ext.googleJavaFormatVersion = '1.18.1'
57-
ext.openLineageVersion = '1.19.0'
60+
ext.openLineageVersion = '1.24.2'
5861
ext.logbackClassicJava8 = '1.2.12'
5962

6063
ext.docker_registry = 'acryldata'
@@ -103,7 +106,14 @@ project.ext.spec = [
103106
]
104107

105108
project.ext.externalDependency = [
106-
'akkaHttp': 'com.typesafe.akka:akka-http-core_2.12:10.2.10',
109+
'akkaHttp': "com.typesafe.akka:akka-http-core_$playScalaVersion:10.2.10", // max version due to licensing
110+
'akkaActor': "com.typesafe.akka:akka-actor_$playScalaVersion:$akkaVersion",
111+
'akkaStream': "com.typesafe.akka:akka-stream_$playScalaVersion:$akkaVersion",
112+
'akkaActorTyped': "com.typesafe.akka:akka-actor-typed_$playScalaVersion:$akkaVersion",
113+
'akkaSlf4j': "com.typesafe.akka:akka-slf4j_$playScalaVersion:$akkaVersion",
114+
'akkaJackson': "com.typesafe.akka:akka-serialization-jackson_$playScalaVersion:$akkaVersion",
115+
'akkaParsing': "com.typesafe.akka:akka-parsing_$playScalaVersion:$akkaVersion",
116+
'akkaProtobuf': "com.typesafe.akka:akka-protobuf-v3_$playScalaVersion:$akkaVersion",
107117
'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3',
108118
'antlr4': 'org.antlr:antlr4:4.9.3',
109119
'assertJ': 'org.assertj:assertj-core:3.11.1',
@@ -212,18 +222,18 @@ project.ext.externalDependency = [
212222
'parquet': 'org.apache.parquet:parquet-avro:1.12.3',
213223
'parquetHadoop': 'org.apache.parquet:parquet-hadoop:1.13.1',
214224
'picocli': 'info.picocli:picocli:4.5.0',
215-
'playCache': "com.typesafe.play:play-cache_2.12:$playVersion",
216-
'playCaffeineCache': "com.typesafe.play:play-caffeine-cache_2.12:$playVersion",
217-
'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.1.10',
218-
'playDocs': "com.typesafe.play:play-docs_2.12:$playVersion",
219-
'playGuice': "com.typesafe.play:play-guice_2.12:$playVersion",
220-
'playJavaJdbc': "com.typesafe.play:play-java-jdbc_2.12:$playVersion",
221-
'playAkkaHttpServer': "com.typesafe.play:play-akka-http-server_2.12:$playVersion",
222-
'playServer': "com.typesafe.play:play-server_2.12:$playVersion",
223-
'playTest': "com.typesafe.play:play-test_2.12:$playVersion",
224-
'playFilters': "com.typesafe.play:filters-helpers_2.12:$playVersion",
225-
'pac4j': 'org.pac4j:pac4j-oidc:4.5.8',
226-
'playPac4j': 'org.pac4j:play-pac4j_2.12:9.0.2',
225+
'playCache': "com.typesafe.play:play-cache_$playScalaVersion:$playVersion",
226+
'playCaffeineCache': "com.typesafe.play:play-caffeine-cache_$playScalaVersion:$playVersion",
227+
'playWs': "com.typesafe.play:play-ahc-ws-standalone_$playScalaVersion:2.1.10",
228+
'playDocs': "com.typesafe.play:play-docs_$playScalaVersion:$playVersion",
229+
'playGuice': "com.typesafe.play:play-guice_$playScalaVersion:$playVersion",
230+
'playJavaJdbc': "com.typesafe.play:play-java-jdbc_$playScalaVersion:$playVersion",
231+
'playAkkaHttpServer': "com.typesafe.play:play-akka-http-server_$playScalaVersion:$playVersion",
232+
'playServer': "com.typesafe.play:play-server_$playScalaVersion:$playVersion",
233+
'playTest': "com.typesafe.play:play-test_$playScalaVersion:$playVersion",
234+
'playFilters': "com.typesafe.play:filters-helpers_$playScalaVersion:$playVersion",
235+
'pac4j': 'org.pac4j:pac4j-oidc:6.0.6',
236+
'playPac4j': "org.pac4j:play-pac4j_$playScalaVersion:12.0.0-PLAY2.8",
227237
'postgresql': 'org.postgresql:postgresql:42.3.9',
228238
'protobuf': 'com.google.protobuf:protobuf-java:3.25.5',
229239
'grpcProtobuf': 'io.grpc:grpc-protobuf:1.53.0',
@@ -348,6 +358,7 @@ allprojects {
348358
}
349359
}
350360
}
361+
351362
}
352363

353364
configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
@@ -391,7 +402,7 @@ subprojects {
391402
implementation externalDependency.annotationApi
392403
constraints {
393404
implementation("com.google.googlejavaformat:google-java-format:$googleJavaFormatVersion")
394-
implementation('io.netty:netty-all:4.1.114.Final')
405+
implementation('io.netty:netty-all:4.1.115.Final')
395406
implementation('org.apache.commons:commons-compress:1.27.1')
396407
implementation('org.apache.velocity:velocity-engine-core:2.4')
397408
implementation('org.hibernate:hibernate-validator:6.0.20.Final')
@@ -407,6 +418,8 @@ subprojects {
407418
googleJavaFormat()
408419
target project.fileTree(project.projectDir) {
409420
include 'src/**/*.java'
421+
include 'app/**/*.java'
422+
include 'test/**/*.java'
410423
exclude 'src/**/resources/'
411424
exclude 'src/**/generated/'
412425
exclude 'src/**/mainGeneratedDataTemplate/'

0 commit comments

Comments
 (0)