Skip to content

Commit 11bc937

Browse files
sgomezvillamorsgomezvillamor
authored
Merge branch 'feature/cus-3571-fix-model-DashboardInfo-DashboardContainsDashboard' into feature/cus-3571/feat-powerbi-report-to-dashboard-lineage
2 parents 18a539e + 00adc98 commit 11bc937

File tree

66 files changed

+656
-278
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+656
-278
lines changed

‎datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductsTab.tsx

+1
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ export default function DataProductsTab() {
6868
searchFlags: { skipCache: true },
6969
},
7070
},
71+
fetchPolicy: 'no-cache',
7172
});
7273
const totalResults = data?.searchAcrossEntities?.total || 0;
7374
const searchResults = data?.searchAcrossEntities?.searchResults?.map((r) => r.entity) || [];

‎docker/datahub-ingestion-base/smoke.Dockerfile

+3-3
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
2020
COPY . /datahub-src
2121
ARG RELEASE_VERSION
2222
RUN cd /datahub-src && \
23-
sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" metadata-ingestion/src/datahub/__init__.py && \
24-
sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/__init__.py && \
25-
cat metadata-ingestion/src/datahub/__init__.py && \
23+
sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" metadata-ingestion/src/datahub/_version.py && \
24+
sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_version.py && \
25+
cat metadata-ingestion/src/datahub/_version.py && \
2626
./gradlew :metadata-ingestion:codegen && \
2727
pip install file:metadata-ingestion-modules/airflow-plugin#egg=acryl-datahub-airflow-plugin file:metadata-ingestion#egg=acryl-datahub
2828

‎docker/datahub-ingestion/Dockerfile

+4-4
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,10 @@ COPY --chown=datahub ./metadata-ingestion-modules/airflow-plugin /metadata-inges
2626

2727
ARG RELEASE_VERSION
2828
WORKDIR /metadata-ingestion
29-
RUN sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/__init__.py && \
30-
sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" airflow-plugin/src/datahub_airflow_plugin/__init__.py && \
31-
cat src/datahub/__init__.py | grep __version__ && \
32-
cat airflow-plugin/src/datahub_airflow_plugin/__init__.py | grep __version__
29+
RUN sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/_version.py && \
30+
sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" airflow-plugin/src/datahub_airflow_plugin/_version.py && \
31+
cat src/datahub/_version.py | grep __version__ && \
32+
cat airflow-plugin/src/datahub_airflow_plugin/_version.py | grep __version__
3333

3434
FROM base AS slim-install
3535

‎docker/datahub-ingestion/Dockerfile-slim-only

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ COPY --chown=datahub ./metadata-ingestion /metadata-ingestion
1515

1616
ARG RELEASE_VERSION
1717
WORKDIR /metadata-ingestion
18-
RUN sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/__init__.py && \
19-
cat src/datahub/__init__.py
18+
RUN sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/_version.py && \
19+
cat src/datahub/_version.py
2020

2121
FROM base as slim-install
2222

‎docs/actions/sources/datahub-cloud-event-source.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ If you've configured your Action pipeline `failure_mode` to be `THROW`, then eve
3838

3939
The DataHub Cloud Event Source produces
4040

41-
- [Entity Change Event V1](../events/entity-change-event.md)
41+
- [Entity Change Event V1](../../managed-datahub/datahub-api/entity-events-api.md)
4242

4343
Note that the DataHub Cloud Event Source does _not_ yet support the full [Metadata Change Log V1](../events/metadata-change-log-event.md) event stream.
4444

‎docs/automations/bigquery-metadata-sync.md

+6
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@ import FeatureAvailability from '@site/src/components/FeatureAvailability';
44

55
<FeatureAvailability saasOnly />
66

7+
:::info
8+
9+
This feature is currently in open beta in Acryl Cloud. Reach out to your Acryl representative to get access.
10+
11+
:::
12+
713
## Introduction
814

915
BigQuery Metadata Sync is an automation that synchronizes DataHub Tags, Table and Column descriptions, and Column Glossary Terms with

‎docs/automations/docs-propagation.md

+6
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
# Documentation Propagation Automation
22

3+
:::info
4+
5+
This feature is currently in open beta in Acryl Cloud. Reach out to your Acryl representative to get access.
6+
7+
:::
8+
39
## Introduction
410

511
Documentation Propagation is an automation automatically propagates column and asset (coming soon) descriptions based on downstream column-level lineage and sibling relationships.

‎docs/automations/glossary-term-propagation.md

+6
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22

33
<FeatureAvailability saasOnly />
44

5+
:::info
6+
7+
This feature is currently in open beta in Acryl Cloud. Reach out to your Acryl representative to get access.
8+
9+
:::
10+
511
## Introduction
612

713
Glossary Term Propagation is an automation feature that propagates classification labels (Glossary Terms) across column and assets based on downstream lineage and sibling relationships.

‎docs/automations/snowflake-tag-propagation.md

+6-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,12 @@ import FeatureAvailability from '@site/src/components/FeatureAvailability';
44

55
<FeatureAvailability saasOnly />
66

7-
> Note that this Automation in currently in open **Beta**. With any questions or issues, please reach out to your Acryl representative.
7+
:::info
8+
9+
This feature is currently in open beta in Acryl Cloud. Reach out to your Acryl representative to get access.
10+
11+
:::
12+
813

914
## Introduction
1015

‎docs/managed-datahub/datahub-api/entity-events-api.md

+1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ The Events API allows you to integrate changes happening on the DataHub Metadata
1515
### Supported Integrations
1616

1717
* [AWS EventBridge](docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge.md)
18+
* [DataHub Cloud Event Source](docs/actions/sources/datahub-cloud-event-source.md)
1819

1920
### Use Cases
2021

‎entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DashboardInfoPatchBuilder.java

+23
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import com.fasterxml.jackson.databind.node.ObjectNode;
1010
import com.linkedin.common.Edge;
1111
import com.linkedin.common.urn.ChartUrn;
12+
import com.linkedin.common.urn.DashboardUrn;
1213
import com.linkedin.common.urn.DatasetUrn;
1314
import com.linkedin.common.urn.Urn;
1415
import com.linkedin.metadata.aspect.patch.PatchOperationType;
@@ -19,6 +20,7 @@ public class DashboardInfoPatchBuilder
1920
extends AbstractMultiFieldPatchBuilder<DashboardInfoPatchBuilder> {
2021
private static final String CHART_EDGES_PATH_START = "/chartEdges/";
2122
private static final String DATASET_EDGES_PATH_START = "/datasetEdges/";
23+
private static final String DASHBOARDS_PATH_START = "/dashboards/";
2224

2325
// Simplified with just Urn
2426
public DashboardInfoPatchBuilder addChartEdge(@Nonnull ChartUrn urn) {
@@ -36,6 +38,7 @@ public DashboardInfoPatchBuilder removeChartEdge(@Nonnull ChartUrn urn) {
3638
return this;
3739
}
3840

41+
// Simplified with just Urn
3942
public DashboardInfoPatchBuilder addDatasetEdge(@Nonnull DatasetUrn urn) {
4043
ObjectNode value = createEdgeValue(urn);
4144

@@ -52,6 +55,22 @@ public DashboardInfoPatchBuilder removeDatasetEdge(@Nonnull DatasetUrn urn) {
5255
return this;
5356
}
5457

58+
// Simplified with just Urn
59+
public DashboardInfoPatchBuilder addDashboard(@Nonnull DashboardUrn urn) {
60+
ObjectNode value = createEdgeValue(urn);
61+
62+
pathValues.add(
63+
ImmutableTriple.of(PatchOperationType.ADD.getValue(), DASHBOARDS_PATH_START + urn, value));
64+
return this;
65+
}
66+
67+
public DashboardInfoPatchBuilder removeDashboard(@Nonnull DashboardUrn urn) {
68+
pathValues.add(
69+
ImmutableTriple.of(
70+
PatchOperationType.REMOVE.getValue(), DASHBOARDS_PATH_START + urn, null));
71+
return this;
72+
}
73+
5574
// Full Edge modification
5675
public DashboardInfoPatchBuilder addEdge(@Nonnull Edge edge) {
5776
ObjectNode value = createEdgeValue(edge);
@@ -87,6 +106,10 @@ private String getEdgePath(@Nonnull Edge edge) {
87106
return CHART_EDGES_PATH_START + destinationUrn;
88107
}
89108

109+
if (DASHBOARD_ENTITY_NAME.equals(destinationUrn.getEntityType())) {
110+
return DASHBOARDS_PATH_START + destinationUrn;
111+
}
112+
90113
// TODO: Output Data Jobs not supported by aspect, add here if this changes
91114

92115
throw new IllegalArgumentException(

‎entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dashboard/DashboardInfoTemplate.java

+13
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ public class DashboardInfoTemplate implements ArrayMergingTemplate<DashboardInfo
2222
private static final String DATASETS_FIELD_NAME = "datasets";
2323
private static final String CHARTS_FIELD_NAME = "charts";
2424
private static final String DESTINATION_URN_FIELD_NAME = "destinationUrn";
25+
private static final String DASHBOARDS_FIELD_NAME = "dashboards";
2526

2627
@Override
2728
public DashboardInfo getSubtype(RecordTemplate recordTemplate) throws ClassCastException {
@@ -74,6 +75,12 @@ public JsonNode transformFields(JsonNode baseNode) {
7475
DATASET_EDGES_FIELD_NAME,
7576
Collections.singletonList(DESTINATION_URN_FIELD_NAME));
7677

78+
transformedNode =
79+
arrayFieldToMap(
80+
transformedNode,
81+
DASHBOARDS_FIELD_NAME,
82+
Collections.singletonList(DESTINATION_URN_FIELD_NAME));
83+
7784
transformedNode =
7885
arrayFieldToMap(transformedNode, DATASETS_FIELD_NAME, Collections.emptyList());
7986

@@ -97,6 +104,12 @@ public JsonNode rebaseFields(JsonNode patched) {
97104
CHART_EDGES_FIELD_NAME,
98105
Collections.singletonList(DESTINATION_URN_FIELD_NAME));
99106

107+
rebasedNode =
108+
transformedMapToArray(
109+
rebasedNode,
110+
DASHBOARDS_FIELD_NAME,
111+
Collections.singletonList(DESTINATION_URN_FIELD_NAME));
112+
100113
rebasedNode = transformedMapToArray(rebasedNode, DATASETS_FIELD_NAME, Collections.emptyList());
101114
rebasedNode = transformedMapToArray(rebasedNode, CHARTS_FIELD_NAME, Collections.emptyList());
102115

‎entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/DashboardInfoTemplateTest.java

+22
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,26 @@ public void testDashboardInfoTemplate() throws Exception {
3131
UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)"),
3232
result.getDatasetEdges().get(0).getDestinationUrn());
3333
}
34+
35+
@Test
36+
public void testDashboardInfoTemplateDashboardsField() throws Exception {
37+
DashboardInfoTemplate dashboardInfoTemplate = new DashboardInfoTemplate();
38+
DashboardInfo dashboardInfo = dashboardInfoTemplate.getDefault();
39+
JsonPatchBuilder jsonPatchBuilder = Json.createPatchBuilder();
40+
jsonPatchBuilder.add(
41+
"/dashboards/urn:li:dashboard:(urn:li:dataPlatform:tableau,SampleDashboard,PROD)",
42+
Json.createObjectBuilder()
43+
.add(
44+
"destinationUrn",
45+
Json.createValue(
46+
"urn:li:dashboard:(urn:li:dataPlatform:tableau,SampleDashboard,PROD)"))
47+
.build());
48+
49+
DashboardInfo result =
50+
dashboardInfoTemplate.applyPatch(dashboardInfo, jsonPatchBuilder.build());
51+
52+
Assert.assertEquals(
53+
UrnUtils.getUrn("urn:li:dashboard:(urn:li:dataPlatform:tableau,SampleDashboard,PROD)"),
54+
result.getDashboards().get(0).getDestinationUrn());
55+
}
3456
}

‎metadata-ingestion-modules/airflow-plugin/.gitignore

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
.envrc
2-
src/datahub_airflow_plugin/__init__.py.bak
32
.vscode/
43
output
54
pvenv36/

‎metadata-ingestion-modules/airflow-plugin/scripts/release.sh

+15-10
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,31 @@
11
#!/bin/bash
2+
# Auto-generated by python-build/generate_release_scripts.py. Do not edit manually.
3+
24
set -euxo pipefail
35

6+
ROOT=../..
7+
MODULE=datahub_airflow_plugin
8+
49
if [[ ! ${RELEASE_SKIP_TEST:-} ]] && [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then
5-
../../gradlew build # also runs tests
10+
${ROOT}/gradlew build # also runs tests
611
elif [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then
7-
../../gradlew install
12+
${ROOT}/gradlew install
813
fi
914

10-
MODULE=datahub_airflow_plugin
11-
1215
# Check packaging constraint.
1316
python -c 'import setuptools; where="./src"; assert setuptools.find_packages(where) == setuptools.find_namespace_packages(where), "you seem to be missing or have extra __init__.py files"'
14-
if [[ ${RELEASE_VERSION:-} ]]; then
15-
# Replace version with RELEASE_VERSION env variable
16-
sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/__init__.py
17-
else
18-
vim src/${MODULE}/__init__.py
17+
18+
# Update the release version.
19+
if [[ ! ${RELEASE_VERSION:-} ]]; then
20+
echo "RELEASE_VERSION is not set"
21+
exit 1
1922
fi
23+
sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/_version.py
2024

25+
# Build and upload the release.
2126
rm -rf build dist || true
2227
python -m build
2328
if [[ ! ${RELEASE_SKIP_UPLOAD:-} ]]; then
2429
python -m twine upload 'dist/*'
2530
fi
26-
mv src/${MODULE}/__init__.py.bak src/${MODULE}/__init__.py
31+
mv src/${MODULE}/_version.py.bak src/${MODULE}/_version.py

‎metadata-ingestion-modules/airflow-plugin/setup.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import setuptools
66

77
package_metadata: dict = {}
8-
with open("./src/datahub_airflow_plugin/__init__.py") as fp:
8+
with open("./src/datahub_airflow_plugin/_version.py") as fp:
99
exec(fp.read(), package_metadata)
1010

1111

‎metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/__init__.py

+4-15
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,12 @@
1-
# Published at https://pypi.org/project/acryl-datahub/.
2-
__package_name__ = "acryl-datahub-airflow-plugin"
3-
__version__ = "1!0.0.0.dev0"
1+
from datahub_airflow_plugin._version import __package_name__, __version__
42

53

6-
def is_dev_mode() -> bool:
7-
return __version__.endswith("dev0")
8-
9-
10-
def nice_version_name() -> str:
11-
if is_dev_mode():
12-
return "unavailable (installed in develop mode)"
13-
return __version__
14-
15-
16-
def get_provider_info():
4+
def get_provider_info() -> dict:
5+
# Register our hooks with Airflow.
176
return {
187
"package-name": f"{__package_name__}",
198
"name": f"{__package_name__}",
20-
"description": "Datahub metadata collector plugin",
9+
"description": "DataHub metadata collector plugin",
2110
"connection-types": [
2211
{
2312
"hook-class-name": "datahub_airflow_plugin.hooks.datahub.DatahubRestHook",

‎metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_version.py

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Published at https://pypi.org/project/acryl-datahub-airflow-plugin/.
2+
__package_name__ = "acryl-datahub-airflow-plugin"
3+
__version__ = "1!0.0.0.dev0"

‎metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@
44

55
from airflow.plugins_manager import AirflowPlugin
66

7-
from datahub_airflow_plugin import __package_name__
87
from datahub_airflow_plugin._airflow_compat import AIRFLOW_PATCHED
98
from datahub_airflow_plugin._airflow_shims import (
109
HAS_AIRFLOW_LISTENER_API,
1110
NEEDS_AIRFLOW_LISTENER_MODULE,
1211
)
12+
from datahub_airflow_plugin._version import __package_name__
1313

1414
assert AIRFLOW_PATCHED
1515
logger = logging.getLogger(__name__)

‎metadata-ingestion-modules/dagster-plugin/.gitignore

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
.envrc
2-
src/datahub_dagster_plugin/__init__.py.bak
32
.vscode/
43
output
54
pvenv36/

‎metadata-ingestion-modules/dagster-plugin/scripts/release.sh

+15-10
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,31 @@
11
#!/bin/bash
2+
# Auto-generated by python-build/generate_release_scripts.py. Do not edit manually.
3+
24
set -euxo pipefail
35

6+
ROOT=../..
7+
MODULE=datahub_dagster_plugin
8+
49
if [[ ! ${RELEASE_SKIP_TEST:-} ]] && [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then
5-
../../gradlew build # also runs tests
10+
${ROOT}/gradlew build # also runs tests
611
elif [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then
7-
../../gradlew install
12+
${ROOT}/gradlew install
813
fi
914

10-
MODULE=datahub_dagster_plugin
11-
1215
# Check packaging constraint.
1316
python -c 'import setuptools; where="./src"; assert setuptools.find_packages(where) == setuptools.find_namespace_packages(where), "you seem to be missing or have extra __init__.py files"'
14-
if [[ ${RELEASE_VERSION:-} ]]; then
15-
# Replace version with RELEASE_VERSION env variable
16-
sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/__init__.py
17-
else
18-
vim src/${MODULE}/__init__.py
17+
18+
# Update the release version.
19+
if [[ ! ${RELEASE_VERSION:-} ]]; then
20+
echo "RELEASE_VERSION is not set"
21+
exit 1
1922
fi
23+
sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/_version.py
2024

25+
# Build and upload the release.
2126
rm -rf build dist || true
2227
python -m build
2328
if [[ ! ${RELEASE_SKIP_UPLOAD:-} ]]; then
2429
python -m twine upload 'dist/*'
2530
fi
26-
mv src/${MODULE}/__init__.py.bak src/${MODULE}/__init__.py
31+
mv src/${MODULE}/_version.py.bak src/${MODULE}/_version.py

‎metadata-ingestion-modules/dagster-plugin/setup.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
import setuptools
55

66
package_metadata: dict = {}
7-
with open("./src/datahub_dagster_plugin/__init__.py") as fp:
7+
with open("./src/datahub_dagster_plugin/_version.py") as fp:
88
exec(fp.read(), package_metadata)
99

1010

0 commit comments

Comments
 (0)