Skip to content

Commit 18ce104

Browse files
david-leifkerdavid-leifker
authored
feat(actions): updates to gha workflows (#11150)
1 parent 897173f commit 18ce104

12 files changed

+93
-45
lines changed

‎.github/actions/ci-optimization/action.yml

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
name: 'Identify CI Optimizations'
2-
description: 'Determine if code changes are specific to certain modules.'
1+
name: "Identify CI Optimizations"
2+
description: "Determine if code changes are specific to certain modules."
33

44
outputs:
55
frontend-only:
@@ -44,9 +44,10 @@ outputs:
4444
runs:
4545
using: "composite"
4646
steps:
47-
- uses: dorny/paths-filter@v2
47+
- uses: dorny/paths-filter@v3
4848
id: filter
4949
with:
50+
token: "" # Empty token forces it to use raw git commands.
5051
filters: |
5152
frontend:
5253
- "datahub-frontend/**"

‎.github/actions/docker-custom-build-and-push/action.yml

+20-11
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,13 @@ inputs:
2626
build-args:
2727
description: "List of build-time variables. Same as docker/build-push-action"
2828
required: false
29-
tags:
30-
# e.g. latest,head,sha12345
31-
description: "List of tags to use for the Docker image"
29+
image_tag:
30+
# e.g. pr12345 OR head OR v0.1.2.3
31+
description: "Main tag to use for the Docker image"
3232
required: true
33+
flavor:
34+
description: 'Image flavor (e.g., slim, full)'
35+
required: false
3336
target:
3437
description: "Sets the target stage to build"
3538
required: false
@@ -45,13 +48,17 @@ runs:
4548
steps:
4649
- name: Docker meta
4750
id: docker_meta
48-
uses: crazy-max/ghaction-docker-meta@v1
51+
uses: docker/metadata-action@v5
4952
with:
50-
# list of Docker images to use as base name for tags
5153
images: ${{ inputs.images }}
52-
# add git short SHA as Docker tag
53-
tag-custom: ${{ inputs.tags }}
54-
tag-custom-only: true
54+
flavor: |
55+
latest=false
56+
suffix=${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }}
57+
tags: |
58+
type=raw,value=${{ inputs.image_tag }}
59+
type=raw,value=head,enable=${{ github.ref == format('refs/heads/{0}', 'acryl-main') }}
60+
type=ref,event=pr,prefix=pr
61+
type=sha,prefix=,format=short
5562
5663
# Code for testing the build when not pushing to Docker Hub.
5764
- name: Build and Load image for testing (if not publishing)
@@ -74,11 +81,13 @@ runs:
7481
if: ${{ inputs.publish != 'true' }}
7582
shell: bash
7683
run: |
84+
IMAGES="""
85+
${{ inputs.images }}
86+
"""
7787
TAGS="""
78-
${{ steps.docker_meta.outputs.tags }}
88+
${{ inputs.image_tag }}
7989
"""
80-
echo "SINGLE_TAG=$(echo $TAGS | tr '\n' ' ' | awk -F' ' '{ print $1 }')" >> $GITHUB_OUTPUT
81-
id: single_tag
90+
echo "SINGLE_TAG=$(echo $IMAGES | tr '\n' ' ' | awk -F' ' '{ print $1 }'):$(echo $TAGS | tr '\n' ' ' | awk -F' ' '{ print $1 }')" >> $GITHUB_OUTPUT
8291
- name: Upload image locally for testing (if not publishing)
8392
uses: ishworkh/docker-image-artifact-upload@v1
8493
if: ${{ inputs.publish != 'true' }}

‎.github/scripts/docker_helpers.sh

+10-2
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@ export MAIN_BRANCH="master"
55
export MAIN_BRANCH_TAG="head"
66

77
function get_short_sha {
8-
echo $(git rev-parse --short "$GITHUB_SHA")
8+
echo $(git rev-parse --short "$GITHUB_SHA"|head -c7)
99
}
1010

1111
export SHORT_SHA=$(get_short_sha)
1212
echo "SHORT_SHA: $SHORT_SHA"
1313

1414
function get_tag {
15-
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g'),${SHORT_SHA}
15+
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
1616
}
1717

1818
function get_tag_slim {
@@ -38,3 +38,11 @@ function get_unique_tag_slim {
3838
function get_unique_tag_full {
3939
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
4040
}
41+
42+
function get_platforms_based_on_branch {
43+
if [ "${{ github.event_name }}" == 'push' && "${{ github.ref }}" == "refs/heads/${MAIN_BRANCH}" ]; then
44+
echo "linux/amd64,linux/arm64"
45+
else
46+
echo "linux/amd64"
47+
fi
48+
}

‎.github/workflows/build-and-test.yml

+5
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,11 @@ jobs:
5757
timeout-minutes: 60
5858
needs: setup
5959
steps:
60+
- name: Free up disk space
61+
run: |
62+
sudo apt-get remove 'dotnet-*' azure-cli || true
63+
sudo rm -rf /usr/local/lib/android/ || true
64+
sudo docker image prune -a -f || true
6065
- uses: szenius/[email protected]
6166
with:
6267
timezoneLinux: ${{ matrix.timezone }}

‎.github/workflows/docker-unified.yml

+25-26
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@ jobs:
4747
publish: ${{ steps.publish.outputs.publish }}
4848
pr-publish: ${{ steps.pr-publish.outputs.publish }}
4949
python_release_version: ${{ steps.tag.outputs.python_release_version }}
50-
short_sha: ${{ steps.tag.outputs.short_sha }}
5150
branch_name: ${{ steps.tag.outputs.branch_name }}
5251
repository_name: ${{ steps.tag.outputs.repository_name }}
5352
frontend_change: ${{ steps.ci-optimize.outputs.frontend-change == 'true' }}
@@ -157,7 +156,7 @@ jobs:
157156
with:
158157
images: |
159158
${{ env.DATAHUB_GMS_IMAGE }}
160-
tags: ${{ needs.setup.outputs.tag }}
159+
image_tag: ${{ needs.setup.outputs.tag }}
161160
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
162161
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
163162
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
@@ -221,7 +220,7 @@ jobs:
221220
with:
222221
images: |
223222
${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}
224-
tags: ${{ needs.setup.outputs.tag }}
223+
image_tag: ${{ needs.setup.outputs.tag }}
225224
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
226225
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
227226
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
@@ -285,7 +284,7 @@ jobs:
285284
with:
286285
images: |
287286
${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}
288-
tags: ${{ needs.setup.outputs.tag }}
287+
image_tag: ${{ needs.setup.outputs.tag }}
289288
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
290289
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
291290
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
@@ -349,7 +348,7 @@ jobs:
349348
with:
350349
images: |
351350
${{ env.DATAHUB_UPGRADE_IMAGE }}
352-
tags: ${{ needs.setup.outputs.tag }}
351+
image_tag: ${{ needs.setup.outputs.tag }}
353352
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
354353
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
355354
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
@@ -394,7 +393,7 @@ jobs:
394393
name: Build and Push DataHub Frontend Docker Image
395394
runs-on: ubuntu-latest
396395
needs: setup
397-
if: ${{ needs.setup.outputs.frontend_change == 'true' || needs.setup.outputs.publish == 'true' }}
396+
if: ${{ needs.setup.outputs.frontend_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true'}}
398397
steps:
399398
- name: Set up JDK 17
400399
uses: actions/setup-java@v3
@@ -415,7 +414,7 @@ jobs:
415414
with:
416415
images: |
417416
${{ env.DATAHUB_FRONTEND_IMAGE }}
418-
tags: ${{ needs.setup.outputs.tag }}
417+
image_tag: ${{ needs.setup.outputs.tag }}
419418
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
420419
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
421420
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
@@ -469,7 +468,7 @@ jobs:
469468
with:
470469
images: |
471470
${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}
472-
tags: ${{ needs.setup.outputs.tag }}
471+
image_tag: ${{ needs.setup.outputs.tag }}
473472
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
474473
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
475474
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
@@ -490,7 +489,7 @@ jobs:
490489
with:
491490
images: |
492491
${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}
493-
tags: ${{ needs.setup.outputs.tag }}
492+
image_tag: ${{ needs.setup.outputs.tag }}
494493
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
495494
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
496495
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
@@ -511,7 +510,7 @@ jobs:
511510
with:
512511
images: |
513512
${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}
514-
tags: ${{ needs.setup.outputs.tag }}
513+
image_tag: ${{ needs.setup.outputs.tag }}
515514
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
516515
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
517516
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
@@ -525,7 +524,7 @@ jobs:
525524
outputs:
526525
tag: ${{ steps.tag.outputs.tag }}
527526
needs: setup
528-
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
527+
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
529528
steps:
530529
- name: Check out the repo
531530
uses: acryldata/sane-checkout-action@v3
@@ -536,7 +535,7 @@ jobs:
536535
target: base
537536
images: |
538537
${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
539-
tags: ${{ needs.setup.outputs.tag }}
538+
image_tag: ${{ needs.setup.outputs.tag }}
540539
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
541540
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
542541
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
@@ -552,7 +551,7 @@ jobs:
552551
outputs:
553552
tag: ${{ steps.tag.outputs.tag }}
554553
needs: [setup, datahub_ingestion_base_build]
555-
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
554+
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
556555
steps:
557556
- name: Check out the repo
558557
uses: acryldata/sane-checkout-action@v3
@@ -574,7 +573,7 @@ jobs:
574573
target: slim-install
575574
images: |
576575
${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
577-
tags: ${{ needs.setup.outputs.slim_tag }}
576+
image_tag: ${{ needs.setup.outputs.slim_tag }}
578577
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
579578
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
580579
build-args: |
@@ -593,7 +592,7 @@ jobs:
593592
outputs:
594593
tag: ${{ steps.tag.outputs.tag }}
595594
needs: [setup, datahub_ingestion_base_build]
596-
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
595+
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
597596
steps:
598597
- name: Check out the repo
599598
uses: acryldata/sane-checkout-action@v3
@@ -636,7 +635,7 @@ jobs:
636635
tag: ${{ steps.tag.outputs.tag }}
637636
needs_artifact_download: ${{ needs.setup.outputs.ingestion_change == 'true' && ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true') }}
638637
needs: [setup, datahub_ingestion_base_slim_build]
639-
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
638+
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
640639
steps:
641640
- name: Set up JDK 17
642641
uses: actions/setup-java@v3
@@ -647,7 +646,7 @@ jobs:
647646
- name: Check out the repo
648647
uses: acryldata/sane-checkout-action@v3
649648
- name: Build codegen
650-
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
649+
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish =='true' }}
651650
run: ./gradlew :metadata-ingestion:codegen
652651
- name: Download Base Image
653652
uses: ishworkh/docker-image-artifact-download@v1
@@ -661,7 +660,7 @@ jobs:
661660
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
662661
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
663662
- name: Build and push Slim Image
664-
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
663+
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
665664
uses: ./.github/actions/docker-custom-build-and-push
666665
with:
667666
target: final
@@ -672,7 +671,7 @@ jobs:
672671
DOCKER_VERSION=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}
673672
RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }}
674673
APP_ENV=slim
675-
tags: ${{ needs.setup.outputs.slim_tag }}
674+
image_tag: ${{ needs.setup.outputs.slim_tag }}
676675
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
677676
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
678677
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
@@ -723,7 +722,7 @@ jobs:
723722
tag: ${{ steps.tag.outputs.tag }}
724723
needs_artifact_download: ${{ needs.setup.outputs.ingestion_change == 'true' && ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' ) }}
725724
needs: [setup, datahub_ingestion_base_full_build]
726-
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
725+
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
727726
steps:
728727
- name: Set up JDK 17
729728
uses: actions/setup-java@v3
@@ -734,7 +733,7 @@ jobs:
734733
- name: Check out the repo
735734
uses: acryldata/sane-checkout-action@v3
736735
- name: Build codegen
737-
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
736+
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
738737
run: ./gradlew :metadata-ingestion:codegen
739738
- name: Download Base Image
740739
uses: ishworkh/docker-image-artifact-download@v1
@@ -748,7 +747,7 @@ jobs:
748747
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
749748
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
750749
- name: Build and push Full Image
751-
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
750+
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
752751
uses: ./.github/actions/docker-custom-build-and-push
753752
with:
754753
target: final
@@ -758,7 +757,7 @@ jobs:
758757
BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
759758
DOCKER_VERSION=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }}
760759
RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }}
761-
tags: ${{ needs.setup.outputs.tag }}
760+
image_tag: ${{ needs.setup.outputs.tag }}
762761
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
763762
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
764763
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
@@ -776,7 +775,7 @@ jobs:
776775
name: "[Monitoring] Scan Datahub Ingestion images for vulnerabilities"
777776
runs-on: ubuntu-latest
778777
needs: [setup, datahub_ingestion_full_build]
779-
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
778+
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
780779
steps:
781780
- name: Checkout # adding checkout step just to make trivy upload happy
782781
uses: acryldata/sane-checkout-action@v3
@@ -965,7 +964,7 @@ jobs:
965964
echo 'datahub-ingestion head-slim images'
966965
docker pull '${{ env.DATAHUB_INGESTION_IMAGE }}:head-slim'
967966
if [ '${{ needs.datahub_ingestion_slim_build.outputs.tag || 'head-slim' }}' != 'head-slim' ]; then
968-
docker tag '${{ env.DATAHUB_INGESTION_IMAGE }}:head-slim' '${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}'
967+
docker tag '${{ env.DATAHUB_INGESTION_IMAGE }}:head-slim' '${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.setup.outputs.unique_tag }}'
969968
fi
970969
fi
971970
- name: Disk Check
@@ -1049,7 +1048,7 @@ jobs:
10491048
runs-on: ubuntu-latest
10501049
needs: [setup, smoke_test]
10511050
steps:
1052-
- uses: aws-actions/configure-aws-credentials@v1
1051+
- uses: aws-actions/configure-aws-credentials@v4
10531052
if: ${{ needs.setup.outputs.publish != 'false' && github.repository_owner == 'datahub-project' && needs.setup.outputs.repository_name == 'datahub' }}
10541053
with:
10551054
aws-access-key-id: ${{ secrets.AWS_SQS_ACCESS_KEY_ID }}

‎.github/workflows/lint-actions.yml

+5
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,8 @@ jobs:
1414
- uses: reviewdog/action-actionlint@v1
1515
with:
1616
reporter: github-pr-review
17+
permissions:
18+
contents: read
19+
checks: write
20+
pull-requests: write
21+
issues: write

‎.github/workflows/metadata-ingestion.yml

+5
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ jobs:
4646
- python-version: "3.10"
4747
fail-fast: false
4848
steps:
49+
- name: Free up disk space
50+
run: |
51+
sudo apt-get remove 'dotnet-*' azure-cli || true
52+
sudo rm -rf /usr/local/lib/android/ || true
53+
sudo docker image prune -a -f || true
4954
- name: Set up JDK 17
5055
uses: actions/setup-java@v3
5156
with:

‎.github/workflows/metadata-io.yml

+5
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,11 @@ jobs:
4747
timeout-minutes: 60
4848
needs: setup
4949
steps:
50+
- name: Free up disk space
51+
run: |
52+
sudo apt-get remove 'dotnet-*' azure-cli || true
53+
sudo rm -rf /usr/local/lib/android/ || true
54+
sudo docker image prune -a -f || true
5055
- uses: acryldata/sane-checkout-action@v3
5156
- name: Set up JDK 17
5257
uses: actions/setup-java@v3

‎.github/workflows/metadata-model.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ jobs:
4949
run: ./gradlew :metadata-ingestion:modelDocGen
5050
- name: Configure AWS Credentials
5151
if: ${{ needs.setup.outputs.publish == 'true' }}
52-
uses: aws-actions/configure-aws-credentials@v3
52+
uses: aws-actions/configure-aws-credentials@v4
5353
with:
5454
aws-access-key-id: ${{ secrets.ACRYL_CI_ARTIFACTS_ACCESS_KEY_ID }}
5555
aws-secret-access-key: ${{ secrets.ACRYL_CI_ARTIFACTS_ACCESS_KEY }}

0 commit comments

Comments
 (0)