Skip to content

Commit f980a86

Browse files
mminichinomminichino
authored
Merge branch 'datahub-project:master' into mm--couchbase-ingest-source
2 parents bb14433 + 5309ae0 commit f980a86

File tree

492 files changed

+10585
-2597
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

492 files changed

+10585
-2597
lines changed

‎.github/dependabot.yml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# To get started with Dependabot version updates, you'll need to specify which
2+
# package ecosystems to update and where the package manifests are located.
3+
# Please see the documentation for all configuration options:
4+
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
5+
6+
version: 2
7+
updates:
8+
- package-ecosystem: "github-actions" # See documentation for possible values
9+
directory: "/" # Location of package manifests
10+
schedule:
11+
interval: "weekly"

‎.github/pr-labeler-config.yml

+23-13
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,32 @@
11
ingestion:
2-
- "metadata-ingestion/**/*"
3-
- "metadata-ingestion-modules/**/*"
4-
- "metadata-integration/**/*"
2+
- changed-files:
3+
- any-glob-to-any-file:
4+
- "metadata-ingestion/**/*"
5+
- "metadata-ingestion-modules/**/*"
6+
- "metadata-integration/**/*"
57

68
devops:
7-
- "docker/**/*"
8-
- ".github/**/*"
9-
- "perf-test/**/*"
10-
- "metadata-service/**/*"
9+
- changed-files:
10+
- any-glob-to-any-file:
11+
- "docker/**/*"
12+
- ".github/**/*"
13+
- "perf-test/**/*"
14+
- "metadata-service/**/*"
1115

1216
product:
13-
- "datahub-web-react/**/*"
14-
- "datahub-frontend/**/*"
15-
- "datahub-graphql-core/**/*"
16-
- "metadata-io/**/*"
17+
- changed-files:
18+
- any-glob-to-any-file:
19+
- "datahub-web-react/**/*"
20+
- "datahub-frontend/**/*"
21+
- "datahub-graphql-core/**/*"
22+
- "metadata-io/**/*"
1723

1824
docs:
19-
- "docs/**/*"
25+
- changed-files:
26+
- any-glob-to-any-file:
27+
- "docs/**/*"
2028

2129
smoke_test:
22-
- "smoke-test/**/*"
30+
- changed-files:
31+
- any-glob-to-any-file:
32+
- "smoke-test/**/*"

‎.github/scripts/pre-commit-override.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,5 @@ repos:
55
name: smoke-test cypress Lint Fix
66
entry: ./gradlew :smoke-test:cypressLintFix
77
language: system
8-
files: ^smoke-test/tests/cypress/.*$
8+
files: ^smoke-test/tests/cypress/.*\.tsx$
99
pass_filenames: false

‎.github/workflows/airflow-plugin.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ jobs:
5656
with:
5757
distribution: "zulu"
5858
java-version: 17
59-
- uses: gradle/actions/setup-gradle@v3
59+
- uses: gradle/actions/setup-gradle@v4
6060
- uses: acryldata/sane-checkout-action@v3
6161
- uses: actions/setup-python@v5
6262
with:

‎.github/workflows/build-and-test.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ jobs:
6363
sudo apt-get remove 'dotnet-*' azure-cli || true
6464
sudo rm -rf /usr/local/lib/android/ || true
6565
sudo docker image prune -a -f || true
66-
- uses: szenius/set-timezone@v1.2
66+
- uses: szenius/set-timezone@v2.0
6767
with:
6868
timezoneLinux: ${{ matrix.timezone }}
6969
- name: Check out the repo
@@ -83,7 +83,7 @@ jobs:
8383
with:
8484
distribution: "zulu"
8585
java-version: 17
86-
- uses: gradle/actions/setup-gradle@v3
86+
- uses: gradle/actions/setup-gradle@v4
8787
- name: Gradle build (and test) for NOT metadata ingestion
8888
if: ${{ matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
8989
# datahub-schematron:cli excluded due to dependency on metadata-ingestion

‎.github/workflows/check-datahub-jars.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ jobs:
4343
with:
4444
distribution: "zulu"
4545
java-version: 17
46-
- uses: gradle/actions/setup-gradle@v3
46+
- uses: gradle/actions/setup-gradle@v4
4747
- name: check ${{ matrix.command }} jar
4848
run: |
4949
./gradlew :metadata-integration:java:${{ matrix.command }}:build --info

‎.github/workflows/dagster-plugin.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ jobs:
4444
with:
4545
distribution: "zulu"
4646
java-version: 17
47-
- uses: gradle/actions/setup-gradle@v3
47+
- uses: gradle/actions/setup-gradle@v4
4848
- uses: acryldata/sane-checkout-action@v3
4949
- uses: actions/setup-python@v5
5050
with:

‎.github/workflows/docker-unified.yml

+31-31
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ jobs:
130130
with:
131131
distribution: "zulu"
132132
java-version: 17
133-
- uses: gradle/actions/setup-gradle@v3
133+
- uses: gradle/actions/setup-gradle@v4
134134
- name: Run lint on smoke test
135135
if: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }}
136136
run: |
@@ -154,7 +154,7 @@ jobs:
154154
with:
155155
distribution: "zulu"
156156
java-version: 17
157-
- uses: gradle/actions/setup-gradle@v3
157+
- uses: gradle/actions/setup-gradle@v4
158158
- name: Check out the repo
159159
uses: acryldata/sane-checkout-action@v3
160160
- name: Pre-build artifacts for docker image
@@ -191,7 +191,7 @@ jobs:
191191
with:
192192
image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
193193
- name: Run Trivy vulnerability scanner
194-
uses: aquasecurity/trivy-action@0.26.0
194+
uses: aquasecurity/trivy-action@0.29.0
195195
env:
196196
TRIVY_OFFLINE_SCAN: true
197197
TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
@@ -205,7 +205,7 @@ jobs:
205205
ignore-unfixed: true
206206
vuln-type: "os,library"
207207
- name: Upload Trivy scan results to GitHub Security tab
208-
uses: github/codeql-action/upload-sarif@v2
208+
uses: github/codeql-action/upload-sarif@v3
209209
with:
210210
sarif_file: "trivy-results.sarif"
211211

@@ -225,7 +225,7 @@ jobs:
225225
with:
226226
distribution: "zulu"
227227
java-version: 17
228-
- uses: gradle/actions/setup-gradle@v3
228+
- uses: gradle/actions/setup-gradle@v4
229229
- name: Check out the repo
230230
uses: acryldata/sane-checkout-action@v3
231231
- name: Pre-build artifacts for docker image
@@ -262,7 +262,7 @@ jobs:
262262
with:
263263
image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
264264
- name: Run Trivy vulnerability scanner
265-
uses: aquasecurity/trivy-action@0.26.0
265+
uses: aquasecurity/trivy-action@0.29.0
266266
env:
267267
TRIVY_OFFLINE_SCAN: true
268268
TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
@@ -276,7 +276,7 @@ jobs:
276276
ignore-unfixed: true
277277
vuln-type: "os,library"
278278
- name: Upload Trivy scan results to GitHub Security tab
279-
uses: github/codeql-action/upload-sarif@v2
279+
uses: github/codeql-action/upload-sarif@v3
280280
with:
281281
sarif_file: "trivy-results.sarif"
282282

@@ -296,7 +296,7 @@ jobs:
296296
with:
297297
distribution: "zulu"
298298
java-version: 17
299-
- uses: gradle/actions/setup-gradle@v3
299+
- uses: gradle/actions/setup-gradle@v4
300300
- name: Check out the repo
301301
uses: acryldata/sane-checkout-action@v3
302302
- name: Pre-build artifacts for docker image
@@ -333,7 +333,7 @@ jobs:
333333
with:
334334
image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
335335
- name: Run Trivy vulnerability scanner
336-
uses: aquasecurity/trivy-action@0.26.0
336+
uses: aquasecurity/trivy-action@0.29.0
337337
env:
338338
TRIVY_OFFLINE_SCAN: true
339339
TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
@@ -347,7 +347,7 @@ jobs:
347347
ignore-unfixed: true
348348
vuln-type: "os,library"
349349
- name: Upload Trivy scan results to GitHub Security tab
350-
uses: github/codeql-action/upload-sarif@v2
350+
uses: github/codeql-action/upload-sarif@v3
351351
with:
352352
sarif_file: "trivy-results.sarif"
353353

@@ -367,7 +367,7 @@ jobs:
367367
with:
368368
distribution: "zulu"
369369
java-version: 17
370-
- uses: gradle/actions/setup-gradle@v3
370+
- uses: gradle/actions/setup-gradle@v4
371371
- name: Check out the repo
372372
uses: acryldata/sane-checkout-action@v3
373373
- name: Pre-build artifacts for docker image
@@ -404,7 +404,7 @@ jobs:
404404
with:
405405
image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
406406
- name: Run Trivy vulnerability scanner
407-
uses: aquasecurity/trivy-action@0.26.0
407+
uses: aquasecurity/trivy-action@0.29.0
408408
env:
409409
TRIVY_OFFLINE_SCAN: true
410410
TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
@@ -418,7 +418,7 @@ jobs:
418418
ignore-unfixed: true
419419
vuln-type: "os,library"
420420
- name: Upload Trivy scan results to GitHub Security tab
421-
uses: github/codeql-action/upload-sarif@v2
421+
uses: github/codeql-action/upload-sarif@v3
422422
with:
423423
sarif_file: "trivy-results.sarif"
424424

@@ -438,7 +438,7 @@ jobs:
438438
with:
439439
distribution: "zulu"
440440
java-version: 17
441-
- uses: gradle/actions/setup-gradle@v3
441+
- uses: gradle/actions/setup-gradle@v4
442442
- name: Check out the repo
443443
uses: acryldata/sane-checkout-action@v3
444444
- name: Pre-build artifacts for docker image
@@ -475,7 +475,7 @@ jobs:
475475
with:
476476
image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
477477
- name: Run Trivy vulnerability scanner
478-
uses: aquasecurity/trivy-action@0.26.0
478+
uses: aquasecurity/trivy-action@0.29.0
479479
env:
480480
TRIVY_OFFLINE_SCAN: true
481481
TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
@@ -489,7 +489,7 @@ jobs:
489489
ignore-unfixed: true
490490
vuln-type: "os,library"
491491
- name: Upload Trivy scan results to GitHub Security tab
492-
uses: github/codeql-action/upload-sarif@v2
492+
uses: github/codeql-action/upload-sarif@v3
493493
with:
494494
sarif_file: "trivy-results.sarif"
495495

@@ -536,7 +536,7 @@ jobs:
536536
with:
537537
image: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
538538
- name: Run Trivy vulnerability scanner
539-
uses: aquasecurity/trivy-action@0.26.0
539+
uses: aquasecurity/trivy-action@0.29.0
540540
env:
541541
TRIVY_OFFLINE_SCAN: true
542542
TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
@@ -550,7 +550,7 @@ jobs:
550550
ignore-unfixed: true
551551
vuln-type: "os,library"
552552
- name: Upload Trivy scan results to GitHub Security tab
553-
uses: github/codeql-action/upload-sarif@v2
553+
uses: github/codeql-action/upload-sarif@v3
554554
with:
555555
sarif_file: "trivy-results.sarif"
556556

@@ -597,7 +597,7 @@ jobs:
597597
with:
598598
image: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
599599
- name: Run Trivy vulnerability scanner
600-
uses: aquasecurity/trivy-action@0.26.0
600+
uses: aquasecurity/trivy-action@0.29.0
601601
env:
602602
TRIVY_OFFLINE_SCAN: true
603603
TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
@@ -611,7 +611,7 @@ jobs:
611611
ignore-unfixed: true
612612
vuln-type: "os,library"
613613
- name: Upload Trivy scan results to GitHub Security tab
614-
uses: github/codeql-action/upload-sarif@v2
614+
uses: github/codeql-action/upload-sarif@v3
615615
with:
616616
sarif_file: "trivy-results.sarif"
617617

@@ -658,7 +658,7 @@ jobs:
658658
with:
659659
image: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
660660
- name: Run Trivy vulnerability scanner
661-
uses: aquasecurity/trivy-action@0.26.0
661+
uses: aquasecurity/trivy-action@0.29.0
662662
env:
663663
TRIVY_OFFLINE_SCAN: true
664664
TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
@@ -672,7 +672,7 @@ jobs:
672672
ignore-unfixed: true
673673
vuln-type: "os,library"
674674
- name: Upload Trivy scan results to GitHub Security tab
675-
uses: github/codeql-action/upload-sarif@v2
675+
uses: github/codeql-action/upload-sarif@v3
676676
with:
677677
sarif_file: "trivy-results.sarif"
678678

@@ -829,7 +829,7 @@ jobs:
829829
with:
830830
distribution: "zulu"
831831
java-version: 17
832-
- uses: gradle/actions/setup-gradle@v3
832+
- uses: gradle/actions/setup-gradle@v4
833833
- name: Build codegen
834834
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish =='true' }}
835835
run: ./gradlew :metadata-ingestion:codegen
@@ -886,7 +886,7 @@ jobs:
886886
with:
887887
image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
888888
- name: Run Trivy vulnerability scanner Slim Image
889-
uses: aquasecurity/trivy-action@0.26.0
889+
uses: aquasecurity/trivy-action@0.29.0
890890
env:
891891
TRIVY_OFFLINE_SCAN: true
892892
TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
@@ -901,7 +901,7 @@ jobs:
901901
vuln-type: "os,library"
902902
timeout: 15m
903903
- name: Upload Trivy scan results to GitHub Security tab
904-
uses: github/codeql-action/upload-sarif@v2
904+
uses: github/codeql-action/upload-sarif@v3
905905
with:
906906
sarif_file: "trivy-results.sarif"
907907

@@ -930,7 +930,7 @@ jobs:
930930
with:
931931
distribution: "zulu"
932932
java-version: 17
933-
- uses: gradle/actions/setup-gradle@v3
933+
- uses: gradle/actions/setup-gradle@v4
934934
- name: Build codegen
935935
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
936936
run: ./gradlew :metadata-ingestion:codegen
@@ -985,7 +985,7 @@ jobs:
985985
with:
986986
image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }}
987987
- name: Run Trivy vulnerability scanner Full Image
988-
uses: aquasecurity/trivy-action@0.26.0
988+
uses: aquasecurity/trivy-action@0.29.0
989989
env:
990990
TRIVY_OFFLINE_SCAN: true
991991
TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
@@ -1000,7 +1000,7 @@ jobs:
10001000
vuln-type: "os,library"
10011001
timeout: 15m
10021002
- name: Upload Trivy scan results to GitHub Security tab
1003-
uses: github/codeql-action/upload-sarif@v2
1003+
uses: github/codeql-action/upload-sarif@v3
10041004
with:
10051005
sarif_file: "trivy-results.sarif"
10061006

@@ -1018,8 +1018,8 @@ jobs:
10181018
# python_batch_count is used to split pytests in the smoke-test (batches of actual test functions)
10191019
# cypress_batch_count is used to split the collection of cypress test specs into batches.
10201020
run: |
1021-
echo "cypress_batch_count=11" >> "$GITHUB_OUTPUT"
1022-
echo "python_batch_count=5" >> "$GITHUB_OUTPUT"
1021+
echo "cypress_batch_count=5" >> "$GITHUB_OUTPUT"
1022+
echo "python_batch_count=3" >> "$GITHUB_OUTPUT"
10231023
10241024
- id: set-matrix
10251025
# For m batches for python and n batches for cypress, we need a test matrix of python x m + cypress x n.
@@ -1083,7 +1083,7 @@ jobs:
10831083
with:
10841084
distribution: "zulu"
10851085
java-version: 17
1086-
- uses: gradle/actions/setup-gradle@v3
1086+
- uses: gradle/actions/setup-gradle@v4
10871087
- name: Login to DockerHub
10881088
uses: docker/login-action@v3
10891089
if: ${{ needs.setup.outputs.docker-login == 'true' }}

0 commit comments

Comments
 (0)