Skip to content

Commit 02f41b7

Browse files
hsheth2hsheth2
authored
feat(ingest): use uv for python package installs (datahub-project#9885)
1 parent a1f2216 commit 02f41b7

File tree

16 files changed

+245
-237
lines changed

16 files changed

+245
-237
lines changed

‎.dockerignore

+5-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
**/node_modules/
22
*/build/
33
*/*/build/
4-
*/venv/
4+
**/venv/
5+
**/.tox/
6+
**/.mypy_cache/
7+
**/.pytest_cache/
8+
**/__pycache__/
59
out
610
**/*.class
711
# Have to copy gradle/wrapper/gradle-wrapper.jar, can't exclude ALL jars

‎.github/scripts/docker_helpers.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ function get_tag_full {
2424
}
2525

2626
function get_python_docker_release_v {
27-
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},0.0.0+docker.${SHORT_SHA},g" -e 's,refs/tags/v\(.*\),\1+docker,g' -e 's,refs/pull/\([0-9]*\).*,0.0.0+docker.pr\1,g')
27+
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},1!0.0.0+docker.${SHORT_SHA},g" -e 's,refs/tags/v\(.*\),1!\1+docker,g' -e 's,refs/pull/\([0-9]*\).*,1!0.0.0+docker.pr\1,g')
2828
}
2929

3030
function get_unique_tag {
@@ -37,4 +37,4 @@ function get_unique_tag_slim {
3737

3838
function get_unique_tag_full {
3939
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
40-
}
40+
}

‎.github/workflows/docker-unified.yml

+14-8
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,17 @@ jobs:
3838
with:
3939
python-version: "3.10"
4040
cache: "pip"
41+
- uses: actions/cache@v4
42+
with:
43+
path: |
44+
~/.cache/uv
45+
key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
4146
- name: Set up JDK 17
4247
uses: actions/setup-java@v3
4348
with:
4449
distribution: "zulu"
4550
java-version: 17
51+
- uses: gradle/gradle-build-action@v2
4652
- name: Ensure packages are correct
4753
run: |
4854
python ./.github/scripts/check_python_package.py
@@ -978,14 +984,14 @@ jobs:
978984
if: failure()
979985
run: |
980986
docker ps -a
981-
docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log || true
982-
docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log || true
983-
docker logs datahub-mae-consumer >& mae-${{ matrix.test_strategy }}.log || true
984-
docker logs datahub-mce-consumer >& mce-${{ matrix.test_strategy }}.log || true
985-
docker logs broker >& broker-${{ matrix.test_strategy }}.log || true
986-
docker logs mysql >& mysql-${{ matrix.test_strategy }}.log || true
987-
docker logs elasticsearch >& elasticsearch-${{ matrix.test_strategy }}.log || true
988-
docker logs datahub-frontend-react >& frontend-${{ matrix.test_strategy }}.log || true
987+
docker logs datahub-datahub-gms-1 >& gms-${{ matrix.test_strategy }}.log || true
988+
docker logs datahub-datahub-actions-1 >& actions-${{ matrix.test_strategy }}.log || true
989+
docker logs datahub-datahub-mae-consumer-1 >& mae-${{ matrix.test_strategy }}.log || true
990+
docker logs datahub-datahub-mce-consumer-1 >& mce-${{ matrix.test_strategy }}.log || true
991+
docker logs datahub-broker-1 >& broker-${{ matrix.test_strategy }}.log || true
992+
docker logs datahub-mysql-1 >& mysql-${{ matrix.test_strategy }}.log || true
993+
docker logs datahub-elasticsearch-1 >& elasticsearch-${{ matrix.test_strategy }}.log || true
994+
docker logs datahub-datahub-frontend-react-1 >& frontend-${{ matrix.test_strategy }}.log || true
989995
- name: Upload logs
990996
uses: actions/upload-artifact@v3
991997
if: failure()

‎.github/workflows/metadata-ingestion.yml

+9
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,15 @@ jobs:
5151
java-version: 17
5252
- uses: gradle/gradle-build-action@v2
5353
- uses: actions/checkout@v3
54+
- uses: actions/setup-python@v4
55+
with:
56+
python-version: ${{ matrix.python-version }}
57+
cache: "pip"
58+
- uses: actions/cache@v4
59+
with:
60+
path: |
61+
~/.cache/uv
62+
key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
5463
- uses: actions/setup-python@v4
5564
with:
5665
python-version: ${{ matrix.python-version }}

‎docker/datahub-ingestion-base/Dockerfile

+14-7
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ RUN apt-get update && apt-get install -y -qq \
5050
ldap-utils \
5151
unixodbc \
5252
libodbc2 \
53-
&& python -m pip install --no-cache --upgrade pip wheel setuptools \
53+
&& python -m pip install --no-cache --upgrade pip uv>=0.1.10 wheel setuptools \
5454
&& rm -rf /var/lib/apt/lists/* /var/cache/apk/*
5555

5656
# compiled against newer golang for security fixes
@@ -59,16 +59,22 @@ COPY --from=dockerize-binary /go/bin/dockerize /usr/local/bin
5959
COPY ./docker/datahub-ingestion-base/base-requirements.txt requirements.txt
6060
COPY ./docker/datahub-ingestion-base/entrypoint.sh /entrypoint.sh
6161

62-
RUN pip install --no-cache -r requirements.txt && \
63-
pip uninstall -y acryl-datahub && \
64-
chmod +x /entrypoint.sh && \
65-
addgroup --gid 1000 datahub && \
66-
adduser --disabled-password --uid 1000 --gid 1000 --home /datahub-ingestion datahub
62+
RUN addgroup --gid 1000 datahub && \
63+
adduser --disabled-password --uid 1000 --gid 1000 --home /datahub-ingestion datahub && \
64+
chmod +x /entrypoint.sh
65+
66+
USER datahub
67+
ENV VIRTUAL_ENV=/datahub-ingestion/.venv
68+
ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
69+
RUN python3 -m venv $VIRTUAL_ENV && \
70+
uv pip install --no-cache -r requirements.txt && \
71+
pip uninstall -y acryl-datahub
6772

6873
ENTRYPOINT [ "/entrypoint.sh" ]
6974

7075
FROM ${BASE_IMAGE} as full-install
7176

77+
USER 0
7278
RUN apt-get update && apt-get install -y -qq \
7379
default-jre-headless \
7480
&& rm -rf /var/lib/apt/lists/* /var/cache/apk/*
@@ -91,10 +97,11 @@ RUN if [ $(arch) = "x86_64" ]; then \
9197
ldconfig; \
9298
fi;
9399

100+
USER datahub
101+
94102
FROM ${BASE_IMAGE} as slim-install
95103
# Do nothing else on top of base
96104

97105
FROM ${APP_ENV}-install
98106

99-
USER datahub
100107
ENV PATH="/datahub-ingestion/.local/bin:$PATH"

0 commit comments

Comments
 (0)