Skip to content

Commit 4cc1706

Browse files
sgomezvillamorsgomezvillamor
authored
Merge branch 'master' into fix-tableau-permission-false-warning
2 parents 182b2f0 + 89acda6 commit 4cc1706

File tree

18 files changed

+524
-60
lines changed

18 files changed

+524
-60
lines changed

‎.github/workflows/airflow-plugin.yml

+2-3
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ jobs:
2727
airflow-plugin:
2828
runs-on: ubuntu-latest
2929
env:
30-
SPARK_VERSION: 3.0.3
3130
DATAHUB_TELEMETRY_ENABLED: false
3231
strategy:
3332
matrix:
@@ -69,7 +68,7 @@ jobs:
6968
- name: pip freeze show list installed
7069
if: always()
7170
run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze
72-
- uses: actions/upload-artifact@v3
71+
- uses: actions/upload-artifact@v4
7372
if: ${{ always() && matrix.python-version == '3.10' && matrix.extra_pip_requirements == 'apache-airflow>=2.7.0' }}
7473
with:
7574
name: Test Results (Airflow Plugin ${{ matrix.python-version}})
@@ -93,7 +92,7 @@ jobs:
9392
runs-on: ubuntu-latest
9493
steps:
9594
- name: Upload
96-
uses: actions/upload-artifact@v3
95+
uses: actions/upload-artifact@v4
9796
with:
9897
name: Event File
9998
path: ${{ github.event_path }}

‎.github/workflows/dagster-plugin.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ jobs:
2727
dagster-plugin:
2828
runs-on: ubuntu-latest
2929
env:
30-
SPARK_VERSION: 3.0.3
3130
DATAHUB_TELEMETRY_ENABLED: false
3231
strategy:
3332
matrix:
@@ -44,7 +43,8 @@ jobs:
4443
with:
4544
distribution: "zulu"
4645
java-version: 17
47-
- uses: actions/checkout@v4
46+
- uses: gradle/actions/setup-gradle@v3
47+
- uses: acryldata/sane-checkout-action@v3
4848
- uses: actions/setup-python@v5
4949
with:
5050
python-version: ${{ matrix.python-version }}
@@ -56,7 +56,7 @@ jobs:
5656
- name: pip freeze show list installed
5757
if: always()
5858
run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && uv pip freeze
59-
- uses: actions/upload-artifact@v3
59+
- uses: actions/upload-artifact@v4
6060
if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }}
6161
with:
6262
name: Test Results (dagster Plugin ${{ matrix.python-version}})
@@ -79,7 +79,7 @@ jobs:
7979
runs-on: ubuntu-latest
8080
steps:
8181
- name: Upload
82-
uses: actions/upload-artifact@v3
82+
uses: actions/upload-artifact@v4
8383
with:
8484
name: Event File
8585
path: ${{ github.event_path }}

‎.github/workflows/gx-plugin.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ jobs:
2727
gx-plugin:
2828
runs-on: ubuntu-latest
2929
env:
30-
SPARK_VERSION: 3.0.3
3130
DATAHUB_TELEMETRY_ENABLED: false
3231
strategy:
3332
matrix:
@@ -48,7 +47,8 @@ jobs:
4847
with:
4948
distribution: "zulu"
5049
java-version: 17
51-
- uses: actions/checkout@v4
50+
- uses: gradle/actions/setup-gradle@v3
51+
- uses: acryldata/sane-checkout-action@v3
5252
- uses: actions/setup-python@v5
5353
with:
5454
python-version: ${{ matrix.python-version }}
@@ -60,7 +60,7 @@ jobs:
6060
- name: pip freeze show list installed
6161
if: always()
6262
run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && uv pip freeze
63-
- uses: actions/upload-artifact@v3
63+
- uses: actions/upload-artifact@v4
6464
if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'great-expectations~=0.17.0' }}
6565
with:
6666
name: Test Results (GX Plugin ${{ matrix.python-version}})
@@ -83,7 +83,7 @@ jobs:
8383
runs-on: ubuntu-latest
8484
steps:
8585
- name: Upload
86-
uses: actions/upload-artifact@v3
86+
uses: actions/upload-artifact@v4
8787
with:
8888
name: Event File
8989
path: ${{ github.event_path }}

‎.github/workflows/metadata-ingestion.yml

+4-5
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ jobs:
2828
runs-on: ubuntu-latest
2929
timeout-minutes: 40
3030
env:
31-
SPARK_VERSION: 3.3.2
3231
DATAHUB_TELEMETRY_ENABLED: false
3332
# TODO: Enable this once the test is fixed.
3433
# DATAHUB_LOOKML_GIT_TEST_SSH_KEY: ${{ secrets.DATAHUB_LOOKML_GIT_TEST_SSH_KEY }}
@@ -84,9 +83,9 @@ jobs:
8483
df -hl
8584
docker image ls
8685
docker system df
87-
- uses: actions/upload-artifact@v3
86+
- uses: actions/upload-artifact@v4
8887
with:
89-
name: Test Results (metadata ingestion ${{ matrix.python-version }})
88+
name: Test Results (metadata ingestion ${{ matrix.python-version }} ${{ matrix.command }})
9089
path: |
9190
**/build/reports/tests/test/**
9291
**/build/test-results/test/**
@@ -100,14 +99,14 @@ jobs:
10099
directory: ./build/coverage-reports/
101100
fail_ci_if_error: false
102101
flags: pytest-${{ matrix.command }}
103-
name: pytest-${{ matrix.command }}
102+
name: pytest-${{ matrix.python-version }}-${{ matrix.command }}
104103
verbose: true
105104

106105
event-file:
107106
runs-on: ubuntu-latest
108107
steps:
109108
- name: Upload
110-
uses: actions/upload-artifact@v3
109+
uses: actions/upload-artifact@v4
111110
with:
112111
name: Event File
113112
path: ${{ github.event_path }}

‎.github/workflows/prefect-plugin.yml

+6-11
Original file line numberDiff line numberDiff line change
@@ -27,25 +27,20 @@ jobs:
2727
prefect-plugin:
2828
runs-on: ubuntu-latest
2929
env:
30-
SPARK_VERSION: 3.0.3
3130
DATAHUB_TELEMETRY_ENABLED: false
3231
strategy:
3332
matrix:
3433
python-version: ["3.8", "3.9", "3.10"]
35-
include:
36-
- python-version: "3.8"
37-
- python-version: "3.9"
38-
- python-version: "3.10"
3934
fail-fast: false
4035
steps:
4136
- name: Set up JDK 17
42-
uses: actions/setup-java@v3
37+
uses: actions/setup-java@v4
4338
with:
4439
distribution: "zulu"
4540
java-version: 17
4641
- uses: gradle/actions/setup-gradle@v3
47-
- uses: actions/checkout@v3
48-
- uses: actions/setup-python@v4
42+
- uses: acryldata/sane-checkout-action@v3
43+
- uses: actions/setup-python@v5
4944
with:
5045
python-version: ${{ matrix.python-version }}
5146
cache: "pip"
@@ -56,7 +51,7 @@ jobs:
5651
- name: pip freeze show list installed
5752
if: always()
5853
run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && uv pip freeze
59-
- uses: actions/upload-artifact@v3
54+
- uses: actions/upload-artifact@v4
6055
if: ${{ always() && matrix.python-version == '3.10'}}
6156
with:
6257
name: Test Results (Prefect Plugin ${{ matrix.python-version}})
@@ -72,15 +67,15 @@ jobs:
7267
token: ${{ secrets.CODECOV_TOKEN }}
7368
directory: ./build/coverage-reports/
7469
fail_ci_if_error: false
75-
flags: prefect,prefect-${{ matrix.extra_pip_extras }}
70+
flags: prefect,prefect-${{ matrix.python-version }}
7671
name: pytest-prefect-${{ matrix.python-version }}
7772
verbose: true
7873

7974
event-file:
8075
runs-on: ubuntu-latest
8176
steps:
8277
- name: Upload
83-
uses: actions/upload-artifact@v3
78+
uses: actions/upload-artifact@v4
8479
with:
8580
name: Event File
8681
path: ${{ github.event_path }}

‎.github/workflows/python-build-pages.yml

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: Python Build
2+
on:
3+
push:
4+
branches:
5+
- master
6+
paths:
7+
- ".github/workflows/python-build-pages.yml"
8+
- "metadata-ingestion/**"
9+
- "metadata-ingestion-modules/**"
10+
- "metadata-models/**"
11+
pull_request:
12+
branches:
13+
- "**"
14+
paths:
15+
- ".github/workflows/python-build-pages.yml"
16+
- "metadata-ingestion/**"
17+
- "metadata-ingestion-modules/**"
18+
- "metadata-models/**"
19+
20+
concurrency:
21+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
22+
cancel-in-progress: true
23+
24+
jobs:
25+
deploy-pages:
26+
runs-on: ubuntu-latest
27+
if: ${{ vars.CLOUDFLARE_WHEELS_PROJECT_NAME != '' }}
28+
29+
name: Python Wheels
30+
permissions:
31+
contents: read
32+
pull-requests: read
33+
deployments: write
34+
steps:
35+
- name: Set up JDK 17
36+
uses: actions/setup-java@v4
37+
with:
38+
distribution: "zulu"
39+
java-version: 17
40+
- uses: gradle/actions/setup-gradle@v3
41+
- uses: acryldata/sane-checkout-action@v3
42+
- uses: actions/setup-python@v5
43+
with:
44+
python-version: "3.10"
45+
cache: "pip"
46+
- uses: actions/cache@v4
47+
with:
48+
path: |
49+
~/.cache/uv
50+
key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
51+
- name: Build Python wheel site
52+
run: |
53+
./gradlew :python-build:buildSite
54+
env:
55+
GITHUB_TOKEN: ${{ github.token }}
56+
- name: Publish
57+
uses: cloudflare/pages-action@v1
58+
with:
59+
apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
60+
accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
61+
projectName: ${{ vars.CLOUDFLARE_WHEELS_PROJECT_NAME }}
62+
workingDirectory: python-build
63+
directory: site
64+
gitHubToken: ${{ github.token }}

‎docs-website/build.gradle

+1-5
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,7 @@ task yarnInstall(type: YarnTask) {
8383
task yarnGenerate(type: YarnTask, dependsOn: [yarnInstall,
8484
generateGraphQLSchema, generateJsonSchema,
8585
':metadata-ingestion:modelDocGen', ':metadata-ingestion:docGen',
86-
':metadata-ingestion:buildWheel',
87-
':metadata-ingestion-modules:airflow-plugin:buildWheel',
88-
':metadata-ingestion-modules:dagster-plugin:buildWheel',
89-
':metadata-ingestion-modules:prefect-plugin:buildWheel',
90-
':metadata-ingestion-modules:gx-plugin:buildWheel',
86+
':python-build:buildWheels',
9187
]) {
9288
inputs.files(projectMdFiles)
9389
outputs.cacheIf { true }

‎docs-website/generateDocsDir.ts

+9-15
Original file line numberDiff line numberDiff line change
@@ -573,26 +573,20 @@ function write_markdown_file(
573573

574574
function copy_python_wheels(): void {
575575
// Copy the built wheel files to the static directory.
576-
const wheel_dirs = [
577-
"../metadata-ingestion/dist",
578-
"../metadata-ingestion-modules/airflow-plugin/dist",
579-
"../metadata-ingestion-modules/dagster-plugin/dist",
580-
"../metadata-ingestion-modules/prefect-plugin/dist",
581-
"../metadata-ingestion-modules/gx-plugin/dist",
582-
];
576+
// Everything is copied to the python-build directory first, so
577+
// we just need to copy from there.
578+
const wheel_dir = "../python-build/wheels";
583579

584580
const wheel_output_directory = path.join(STATIC_DIRECTORY, "wheels");
585581
fs.mkdirSync(wheel_output_directory, { recursive: true });
586582

587-
for (const wheel_dir of wheel_dirs) {
588-
const wheel_files = fs.readdirSync(wheel_dir);
589-
for (const wheel_file of wheel_files) {
590-
const src = path.join(wheel_dir, wheel_file);
591-
const dest = path.join(wheel_output_directory, wheel_file);
583+
const wheel_files = fs.readdirSync(wheel_dir);
584+
for (const wheel_file of wheel_files) {
585+
const src = path.join(wheel_dir, wheel_file);
586+
const dest = path.join(wheel_output_directory, wheel_file);
592587

593-
// console.log(`Copying artifact ${src} to ${dest}...`);
594-
fs.copyFileSync(src, dest);
595-
}
588+
// console.log(`Copying artifact ${src} to ${dest}...`);
589+
fs.copyFileSync(src, dest);
596590
}
597591
}
598592

‎docs/managed-datahub/release-notes/v_0_3_7.md

+31
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,43 @@ If you are using an older CLI/SDK version, then please upgrade it. This applies
1313

1414
## Known Issues
1515

16+
### v0.3.7.7
17+
* Postgres regression, non-functional when using postgres
18+
1619
### v0.3.7.3
1720
* Search page fails to render when filters are applied with a query which returns zero results.
1821

1922
## Release Changelog
2023
---
2124

25+
### v0.3.7.8
26+
27+
- [Postgres] Fix regression from MySQL fix in v0.3.7.7
28+
29+
### v0.3.7.7
30+
31+
- [UI] Fix bug showing upstream lineage dbt source leaves
32+
- [UI] Show column-level lineage through transformational home node
33+
- [UI] Browse nodes titles expand to full width of panel
34+
- [UI] Data product preview cards display correctly
35+
- [UI] Fix elasticsearch usage sort field names
36+
- [UI] Add structured property display settings feature
37+
- [Executor] Fix false errors on cli ingestions
38+
- [Search] Schema field boost reduced
39+
- [Search] Search usage ranking null_fill fix
40+
- [Search] Single term with underscores by default no longer considered quoted
41+
- [Metadata Tests] Metadata Test shutdown actions flush
42+
- [Metadata Tests] Add deduplicate logic for MCP batches
43+
- [Metadata Tests] Prevent mutation of systemMetadata in patch batches
44+
- [MAE Consumer] Fix graph edge on container delete exception
45+
- [Notifications] Filter out system ingestion source notifications
46+
- [MySQL] Fix index gap lock deadlock
47+
- [API] DataJobInputOutput finegrained lineage fix
48+
49+
### v0.3.7.6
50+
51+
- [UI] fix(automations): white screen automations with dbt sync
52+
2253
### v0.3.7.5
2354

2455
- [GMS] Fix upstream lineage patching when path contained encoded slash

‎metadata-ingestion/build.gradle

+2-2
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
2323
inputs.file file('setup.py')
2424
outputs.file(sentinel_file)
2525
commandLine 'bash', '-c',
26-
"${python_executable} -m venv ${venv_name} && " +
27-
"${venv_name}/bin/python -m pip install --upgrade pip uv wheel 'setuptools>=63.0.0' && " +
26+
"${python_executable} -m venv ${venv_name} && set -x && " +
27+
"${venv_name}/bin/python -m pip install --upgrade uv && " +
2828
"touch ${sentinel_file}"
2929
}
3030

0 commit comments

Comments
 (0)