Skip to content

Commit c53d647

Browse files
sid-acrylsid-acryl
authored
Merge branch 'master' into remove-page-size
2 parents c409a83 + acb76cd commit c53d647

File tree

54 files changed

+1541
-496
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+1541
-496
lines changed

‎.github/workflows/airflow-plugin.yml

+2-3
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ jobs:
2727
airflow-plugin:
2828
runs-on: ubuntu-latest
2929
env:
30-
SPARK_VERSION: 3.0.3
3130
DATAHUB_TELEMETRY_ENABLED: false
3231
strategy:
3332
matrix:
@@ -69,7 +68,7 @@ jobs:
6968
- name: pip freeze show list installed
7069
if: always()
7170
run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze
72-
- uses: actions/upload-artifact@v3
71+
- uses: actions/upload-artifact@v4
7372
if: ${{ always() && matrix.python-version == '3.10' && matrix.extra_pip_requirements == 'apache-airflow>=2.7.0' }}
7473
with:
7574
name: Test Results (Airflow Plugin ${{ matrix.python-version}})
@@ -93,7 +92,7 @@ jobs:
9392
runs-on: ubuntu-latest
9493
steps:
9594
- name: Upload
96-
uses: actions/upload-artifact@v3
95+
uses: actions/upload-artifact@v4
9796
with:
9897
name: Event File
9998
path: ${{ github.event_path }}

‎.github/workflows/dagster-plugin.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ jobs:
2727
dagster-plugin:
2828
runs-on: ubuntu-latest
2929
env:
30-
SPARK_VERSION: 3.0.3
3130
DATAHUB_TELEMETRY_ENABLED: false
3231
strategy:
3332
matrix:
@@ -44,7 +43,8 @@ jobs:
4443
with:
4544
distribution: "zulu"
4645
java-version: 17
47-
- uses: actions/checkout@v4
46+
- uses: gradle/actions/setup-gradle@v3
47+
- uses: acryldata/sane-checkout-action@v3
4848
- uses: actions/setup-python@v5
4949
with:
5050
python-version: ${{ matrix.python-version }}
@@ -56,7 +56,7 @@ jobs:
5656
- name: pip freeze show list installed
5757
if: always()
5858
run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && uv pip freeze
59-
- uses: actions/upload-artifact@v3
59+
- uses: actions/upload-artifact@v4
6060
if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }}
6161
with:
6262
name: Test Results (dagster Plugin ${{ matrix.python-version}})
@@ -79,7 +79,7 @@ jobs:
7979
runs-on: ubuntu-latest
8080
steps:
8181
- name: Upload
82-
uses: actions/upload-artifact@v3
82+
uses: actions/upload-artifact@v4
8383
with:
8484
name: Event File
8585
path: ${{ github.event_path }}

‎.github/workflows/gx-plugin.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ jobs:
2727
gx-plugin:
2828
runs-on: ubuntu-latest
2929
env:
30-
SPARK_VERSION: 3.0.3
3130
DATAHUB_TELEMETRY_ENABLED: false
3231
strategy:
3332
matrix:
@@ -48,7 +47,8 @@ jobs:
4847
with:
4948
distribution: "zulu"
5049
java-version: 17
51-
- uses: actions/checkout@v4
50+
- uses: gradle/actions/setup-gradle@v3
51+
- uses: acryldata/sane-checkout-action@v3
5252
- uses: actions/setup-python@v5
5353
with:
5454
python-version: ${{ matrix.python-version }}
@@ -60,7 +60,7 @@ jobs:
6060
- name: pip freeze show list installed
6161
if: always()
6262
run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && uv pip freeze
63-
- uses: actions/upload-artifact@v3
63+
- uses: actions/upload-artifact@v4
6464
if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'great-expectations~=0.17.0' }}
6565
with:
6666
name: Test Results (GX Plugin ${{ matrix.python-version}})
@@ -83,7 +83,7 @@ jobs:
8383
runs-on: ubuntu-latest
8484
steps:
8585
- name: Upload
86-
uses: actions/upload-artifact@v3
86+
uses: actions/upload-artifact@v4
8787
with:
8888
name: Event File
8989
path: ${{ github.event_path }}

‎.github/workflows/metadata-ingestion.yml

+4-5
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ jobs:
2828
runs-on: ubuntu-latest
2929
timeout-minutes: 40
3030
env:
31-
SPARK_VERSION: 3.3.2
3231
DATAHUB_TELEMETRY_ENABLED: false
3332
# TODO: Enable this once the test is fixed.
3433
# DATAHUB_LOOKML_GIT_TEST_SSH_KEY: ${{ secrets.DATAHUB_LOOKML_GIT_TEST_SSH_KEY }}
@@ -84,9 +83,9 @@ jobs:
8483
df -hl
8584
docker image ls
8685
docker system df
87-
- uses: actions/upload-artifact@v3
86+
- uses: actions/upload-artifact@v4
8887
with:
89-
name: Test Results (metadata ingestion ${{ matrix.python-version }})
88+
name: Test Results (metadata ingestion ${{ matrix.python-version }} ${{ matrix.command }})
9089
path: |
9190
**/build/reports/tests/test/**
9291
**/build/test-results/test/**
@@ -100,14 +99,14 @@ jobs:
10099
directory: ./build/coverage-reports/
101100
fail_ci_if_error: false
102101
flags: pytest-${{ matrix.command }}
103-
name: pytest-${{ matrix.command }}
102+
name: pytest-${{ matrix.python-version }}-${{ matrix.command }}
104103
verbose: true
105104

106105
event-file:
107106
runs-on: ubuntu-latest
108107
steps:
109108
- name: Upload
110-
uses: actions/upload-artifact@v3
109+
uses: actions/upload-artifact@v4
111110
with:
112111
name: Event File
113112
path: ${{ github.event_path }}

‎.github/workflows/prefect-plugin.yml

+6-11
Original file line numberDiff line numberDiff line change
@@ -27,25 +27,20 @@ jobs:
2727
prefect-plugin:
2828
runs-on: ubuntu-latest
2929
env:
30-
SPARK_VERSION: 3.0.3
3130
DATAHUB_TELEMETRY_ENABLED: false
3231
strategy:
3332
matrix:
3433
python-version: ["3.8", "3.9", "3.10"]
35-
include:
36-
- python-version: "3.8"
37-
- python-version: "3.9"
38-
- python-version: "3.10"
3934
fail-fast: false
4035
steps:
4136
- name: Set up JDK 17
42-
uses: actions/setup-java@v3
37+
uses: actions/setup-java@v4
4338
with:
4439
distribution: "zulu"
4540
java-version: 17
4641
- uses: gradle/actions/setup-gradle@v3
47-
- uses: actions/checkout@v3
48-
- uses: actions/setup-python@v4
42+
- uses: acryldata/sane-checkout-action@v3
43+
- uses: actions/setup-python@v5
4944
with:
5045
python-version: ${{ matrix.python-version }}
5146
cache: "pip"
@@ -56,7 +51,7 @@ jobs:
5651
- name: pip freeze show list installed
5752
if: always()
5853
run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && uv pip freeze
59-
- uses: actions/upload-artifact@v3
54+
- uses: actions/upload-artifact@v4
6055
if: ${{ always() && matrix.python-version == '3.10'}}
6156
with:
6257
name: Test Results (Prefect Plugin ${{ matrix.python-version}})
@@ -72,15 +67,15 @@ jobs:
7267
token: ${{ secrets.CODECOV_TOKEN }}
7368
directory: ./build/coverage-reports/
7469
fail_ci_if_error: false
75-
flags: prefect,prefect-${{ matrix.extra_pip_extras }}
70+
flags: prefect,prefect-${{ matrix.python-version }}
7671
name: pytest-prefect-${{ matrix.python-version }}
7772
verbose: true
7873

7974
event-file:
8075
runs-on: ubuntu-latest
8176
steps:
8277
- name: Upload
83-
uses: actions/upload-artifact@v3
78+
uses: actions/upload-artifact@v4
8479
with:
8580
name: Event File
8681
path: ${{ github.event_path }}

‎.github/workflows/python-build-pages.yml

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: Python Build
2+
on:
3+
push:
4+
branches:
5+
- master
6+
paths:
7+
- ".github/workflows/python-build-pages.yml"
8+
- "metadata-ingestion/**"
9+
- "metadata-ingestion-modules/**"
10+
- "metadata-models/**"
11+
pull_request:
12+
branches:
13+
- "**"
14+
paths:
15+
- ".github/workflows/python-build-pages.yml"
16+
- "metadata-ingestion/**"
17+
- "metadata-ingestion-modules/**"
18+
- "metadata-models/**"
19+
20+
concurrency:
21+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
22+
cancel-in-progress: true
23+
24+
jobs:
25+
deploy-pages:
26+
runs-on: ubuntu-latest
27+
if: ${{ vars.CLOUDFLARE_WHEELS_PROJECT_NAME != '' }}
28+
29+
name: Python Wheels
30+
permissions:
31+
contents: read
32+
pull-requests: read
33+
deployments: write
34+
steps:
35+
- name: Set up JDK 17
36+
uses: actions/setup-java@v4
37+
with:
38+
distribution: "zulu"
39+
java-version: 17
40+
- uses: gradle/actions/setup-gradle@v3
41+
- uses: acryldata/sane-checkout-action@v3
42+
- uses: actions/setup-python@v5
43+
with:
44+
python-version: "3.10"
45+
cache: "pip"
46+
- uses: actions/cache@v4
47+
with:
48+
path: |
49+
~/.cache/uv
50+
key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
51+
- name: Build Python wheel site
52+
run: |
53+
./gradlew :python-build:buildSite
54+
env:
55+
GITHUB_TOKEN: ${{ github.token }}
56+
- name: Publish
57+
uses: cloudflare/pages-action@v1
58+
with:
59+
apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
60+
accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
61+
projectName: ${{ vars.CLOUDFLARE_WHEELS_PROJECT_NAME }}
62+
workingDirectory: python-build
63+
directory: site
64+
gitHubToken: ${{ github.token }}

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/EmbeddedModelMapper.java

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package com.linkedin.datahub.graphql.types.mappers;
2+
3+
import com.linkedin.common.urn.Urn;
4+
import com.linkedin.datahub.graphql.QueryContext;
5+
import javax.annotation.Nonnull;
6+
import javax.annotation.Nullable;
7+
8+
/** Made for models that are embedded in other models and thus do not encode their own URN. */
9+
public interface EmbeddedModelMapper<I, O> {
10+
O apply(
11+
@Nullable final QueryContext context, @Nonnull final I input, @Nonnull final Urn entityUrn);
12+
}

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java

+8-4
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,8 @@ public MLFeature apply(
7575
mlFeature.setOwnership(
7676
OwnershipMapper.map(context, new Ownership(dataMap), entityUrn)));
7777
mappingHelper.mapToResult(
78-
context, ML_FEATURE_PROPERTIES_ASPECT_NAME, MLFeatureMapper::mapMLFeatureProperties);
78+
ML_FEATURE_PROPERTIES_ASPECT_NAME,
79+
(entity, dataMap) -> mapMLFeatureProperties(context, entity, dataMap, entityUrn));
7980
mappingHelper.mapToResult(
8081
INSTITUTIONAL_MEMORY_ASPECT_NAME,
8182
(mlFeature, dataMap) ->
@@ -138,10 +139,13 @@ private static void mapMLFeatureKey(@Nonnull MLFeature mlFeature, @Nonnull DataM
138139
private static void mapMLFeatureProperties(
139140
@Nullable final QueryContext context,
140141
@Nonnull MLFeature mlFeature,
141-
@Nonnull DataMap dataMap) {
142+
@Nonnull DataMap dataMap,
143+
@Nonnull Urn entityUrn) {
142144
MLFeatureProperties featureProperties = new MLFeatureProperties(dataMap);
143-
mlFeature.setFeatureProperties(MLFeaturePropertiesMapper.map(context, featureProperties));
144-
mlFeature.setProperties(MLFeaturePropertiesMapper.map(context, featureProperties));
145+
com.linkedin.datahub.graphql.generated.MLFeatureProperties graphqlProperties =
146+
MLFeaturePropertiesMapper.map(context, featureProperties, entityUrn);
147+
mlFeature.setFeatureProperties(graphqlProperties);
148+
mlFeature.setProperties(graphqlProperties);
145149
mlFeature.setDescription(featureProperties.getDescription());
146150
if (featureProperties.getDataType() != null) {
147151
mlFeature.setDataType(MLFeatureDataType.valueOf(featureProperties.getDataType().toString()));

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeaturePropertiesMapper.java

+14-6
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,34 @@
11
package com.linkedin.datahub.graphql.types.mlmodel.mappers;
22

3+
import com.linkedin.common.urn.Urn;
34
import com.linkedin.datahub.graphql.QueryContext;
45
import com.linkedin.datahub.graphql.generated.Dataset;
56
import com.linkedin.datahub.graphql.generated.MLFeatureDataType;
67
import com.linkedin.datahub.graphql.generated.MLFeatureProperties;
7-
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
8+
import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
9+
import com.linkedin.datahub.graphql.types.mappers.EmbeddedModelMapper;
810
import java.util.stream.Collectors;
11+
import javax.annotation.Nonnull;
912
import javax.annotation.Nullable;
10-
import lombok.NonNull;
1113

1214
public class MLFeaturePropertiesMapper
13-
implements ModelMapper<com.linkedin.ml.metadata.MLFeatureProperties, MLFeatureProperties> {
15+
implements EmbeddedModelMapper<
16+
com.linkedin.ml.metadata.MLFeatureProperties, MLFeatureProperties> {
1417

1518
public static final MLFeaturePropertiesMapper INSTANCE = new MLFeaturePropertiesMapper();
1619

1720
public static MLFeatureProperties map(
1821
@Nullable QueryContext context,
19-
@NonNull final com.linkedin.ml.metadata.MLFeatureProperties mlFeatureProperties) {
20-
return INSTANCE.apply(context, mlFeatureProperties);
22+
@Nonnull final com.linkedin.ml.metadata.MLFeatureProperties mlFeatureProperties,
23+
@Nonnull Urn entityUrn) {
24+
return INSTANCE.apply(context, mlFeatureProperties, entityUrn);
2125
}
2226

2327
@Override
2428
public MLFeatureProperties apply(
2529
@Nullable QueryContext context,
26-
@NonNull final com.linkedin.ml.metadata.MLFeatureProperties mlFeatureProperties) {
30+
@Nonnull final com.linkedin.ml.metadata.MLFeatureProperties mlFeatureProperties,
31+
@Nonnull Urn entityUrn) {
2732
final MLFeatureProperties result = new MLFeatureProperties();
2833

2934
result.setDescription(mlFeatureProperties.getDescription());
@@ -45,6 +50,9 @@ public MLFeatureProperties apply(
4550
.collect(Collectors.toList()));
4651
}
4752

53+
result.setCustomProperties(
54+
CustomPropertiesMapper.map(mlFeatureProperties.getCustomProperties(), entityUrn));
55+
4856
return result;
4957
}
5058
}

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ public MLFeatureTable apply(
7676
mappingHelper.mapToResult(ML_FEATURE_TABLE_KEY_ASPECT_NAME, this::mapMLFeatureTableKey);
7777
mappingHelper.mapToResult(
7878
ML_FEATURE_TABLE_PROPERTIES_ASPECT_NAME,
79-
(entity, dataMap) -> this.mapMLFeatureTableProperties(context, entity, dataMap, entityUrn));
79+
(entity, dataMap) -> mapMLFeatureTableProperties(context, entity, dataMap, entityUrn));
8080
mappingHelper.mapToResult(
8181
INSTITUTIONAL_MEMORY_ASPECT_NAME,
8282
(mlFeatureTable, dataMap) ->
@@ -146,10 +146,10 @@ private static void mapMLFeatureTableProperties(
146146
@Nonnull DataMap dataMap,
147147
Urn entityUrn) {
148148
MLFeatureTableProperties featureTableProperties = new MLFeatureTableProperties(dataMap);
149-
mlFeatureTable.setFeatureTableProperties(
150-
MLFeatureTablePropertiesMapper.map(context, featureTableProperties, entityUrn));
151-
mlFeatureTable.setProperties(
152-
MLFeatureTablePropertiesMapper.map(context, featureTableProperties, entityUrn));
149+
com.linkedin.datahub.graphql.generated.MLFeatureTableProperties graphqlProperties =
150+
MLFeatureTablePropertiesMapper.map(context, featureTableProperties, entityUrn);
151+
mlFeatureTable.setFeatureTableProperties(graphqlProperties);
152+
mlFeatureTable.setProperties(graphqlProperties);
153153
mlFeatureTable.setDescription(featureTableProperties.getDescription());
154154
}
155155

0 commit comments

Comments
 (0)