fix(model): fixes DashboardContainsDashboard relationship in DashboardInfo aspect (#12433)

Author: sgomezvillamor

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/ReindexDashboardInfoConfig.java

@@ -0,0 +1,30 @@
+package com.linkedin.datahub.upgrade.config.restoreindices;
+
+import com.linkedin.datahub.upgrade.config.SystemUpdateCondition;
+import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
+import com.linkedin.datahub.upgrade.system.restoreindices.dashboardinfo.ReindexDashboardInfo;
+import com.linkedin.metadata.entity.AspectDao;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Conditional;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@Conditional(SystemUpdateCondition.NonBlockingSystemUpdateCondition.class)
+public class ReindexDashboardInfoConfig {
+
+  @Bean
+  public NonBlockingSystemUpgrade reindexDashboardInfo(
+      final OperationContext opContext,
+      final EntityService<?> entityService,
+      final AspectDao aspectDao,
+      @Value("${systemUpdate.dashboardInfo.enabled}") final boolean enabled,
+      @Value("${systemUpdate.dashboardInfo.batchSize}") final Integer batchSize,
+      @Value("${systemUpdate.dashboardInfo.delayMs}") final Integer delayMs,
+      @Value("${systemUpdate.dashboardInfo.limit}") final Integer limit) {
+    return new ReindexDashboardInfo(
+        opContext, entityService, aspectDao, enabled, batchSize, delayMs, limit);
+  }
+}

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/dashboardinfo/ReindexDashboardInfo.java

@@ -0,0 +1,49 @@
+package com.linkedin.datahub.upgrade.system.restoreindices.dashboardinfo;
+
+import com.google.common.collect.ImmutableList;
+import com.linkedin.datahub.upgrade.UpgradeStep;
+import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
+import com.linkedin.metadata.entity.AspectDao;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import java.util.List;
+import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * A job that reindexes all dashboard info aspects as part of reindexing dashboards relationship.
+ * This is required to fix the dashboards relationships for dashboards
+ */
+@Slf4j
+public class ReindexDashboardInfo implements NonBlockingSystemUpgrade {
+
+  private final List<UpgradeStep> _steps;
+
+  public ReindexDashboardInfo(
+      @Nonnull OperationContext opContext,
+      EntityService<?> entityService,
+      AspectDao aspectDao,
+      boolean enabled,
+      Integer batchSize,
+      Integer batchDelayMs,
+      Integer limit) {
+    if (enabled) {
+      _steps =
+          ImmutableList.of(
+              new ReindexDashboardInfoStep(
+                  opContext, entityService, aspectDao, batchSize, batchDelayMs, limit));
+    } else {
+      _steps = ImmutableList.of();
+    }
+  }
+
+  @Override
+  public String id() {
+    return this.getClass().getName();
+  }
+
+  @Override
+  public List<UpgradeStep> steps() {
+    return _steps;
+  }
+}

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/dashboardinfo/ReindexDashboardInfoStep.java

@@ -0,0 +1,42 @@
+package com.linkedin.datahub.upgrade.system.restoreindices.dashboardinfo;
+
+import static com.linkedin.metadata.Constants.*;
+
+import com.linkedin.datahub.upgrade.system.AbstractMCLStep;
+import com.linkedin.metadata.entity.AspectDao;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
+import org.jetbrains.annotations.Nullable;
+
+@Slf4j
+public class ReindexDashboardInfoStep extends AbstractMCLStep {
+
+  public ReindexDashboardInfoStep(
+      OperationContext opContext,
+      EntityService<?> entityService,
+      AspectDao aspectDao,
+      Integer batchSize,
+      Integer batchDelayMs,
+      Integer limit) {
+    super(opContext, entityService, aspectDao, batchSize, batchDelayMs, limit);
+  }
+
+  @Override
+  public String id() {
+    return "dashboard-info-v1";
+  }
+
+  @Nonnull
+  @Override
+  protected String getAspectName() {
+    return DASHBOARD_INFO_ASPECT_NAME;
+  }
+
+  @Nullable
+  @Override
+  protected String getUrnLike() {
+    return "urn:li:" + DASHBOARD_ENTITY_NAME + ":%";
+  }
+}

docs/how/updating-datahub.md

@@ -28,6 +28,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 ### Other Notable Changes
 
+- #12433: Fixes the searchable annotations in the model supporting `Dashboard` to `Dashboard` lineage within the `DashboardInfo` aspect. Mainly, users of Sigma and PowerBI Apps ingestion may be affected by this adjustment. Consequently, a [reindex](https://datahubproject.io/docs/how/restore-indices/) will be automatically triggered during the system upgrade.
+
 ## 0.15.0
 
 - OpenAPI Update: PIT Keep Alive parameter added to scroll endpoints. NOTE: This parameter requires the `pointInTimeCreationEnabled` feature flag to be enabled and the `elasticSearch.implementation` configuration to be `elasticsearch`. This feature is not supported for OpenSearch at this time and the parameter will not be respected without both of these set.

entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DashboardInfoPatchBuilder.java

@@ -9,6 +9,7 @@
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.linkedin.common.Edge;
 import com.linkedin.common.urn.ChartUrn;
+import com.linkedin.common.urn.DashboardUrn;
 import com.linkedin.common.urn.DatasetUrn;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.metadata.aspect.patch.PatchOperationType;
@@ -19,6 +20,7 @@ public class DashboardInfoPatchBuilder
     extends AbstractMultiFieldPatchBuilder<DashboardInfoPatchBuilder> {
   private static final String CHART_EDGES_PATH_START = "/chartEdges/";
   private static final String DATASET_EDGES_PATH_START = "/datasetEdges/";
+  private static final String DASHBOARDS_PATH_START = "/dashboards/";
 
   // Simplified with just Urn
   public DashboardInfoPatchBuilder addChartEdge(@Nonnull ChartUrn urn) {
@@ -36,6 +38,7 @@ public DashboardInfoPatchBuilder removeChartEdge(@Nonnull ChartUrn urn) {
     return this;
   }
 
+  // Simplified with just Urn
   public DashboardInfoPatchBuilder addDatasetEdge(@Nonnull DatasetUrn urn) {
     ObjectNode value = createEdgeValue(urn);
 
@@ -52,6 +55,22 @@ public DashboardInfoPatchBuilder removeDatasetEdge(@Nonnull DatasetUrn urn) {
     return this;
   }
 
+  // Simplified with just Urn
+  public DashboardInfoPatchBuilder addDashboard(@Nonnull DashboardUrn urn) {
+    ObjectNode value = createEdgeValue(urn);
+
+    pathValues.add(
+        ImmutableTriple.of(PatchOperationType.ADD.getValue(), DASHBOARDS_PATH_START + urn, value));
+    return this;
+  }
+
+  public DashboardInfoPatchBuilder removeDashboard(@Nonnull DashboardUrn urn) {
+    pathValues.add(
+        ImmutableTriple.of(
+            PatchOperationType.REMOVE.getValue(), DASHBOARDS_PATH_START + urn, null));
+    return this;
+  }
+
   // Full Edge modification
   public DashboardInfoPatchBuilder addEdge(@Nonnull Edge edge) {
     ObjectNode value = createEdgeValue(edge);
@@ -87,6 +106,10 @@ private String getEdgePath(@Nonnull Edge edge) {
       return CHART_EDGES_PATH_START + destinationUrn;
     }
 
+    if (DASHBOARD_ENTITY_NAME.equals(destinationUrn.getEntityType())) {
+      return DASHBOARDS_PATH_START + destinationUrn;
+    }
+
     // TODO: Output Data Jobs not supported by aspect, add here if this changes
 
     throw new IllegalArgumentException(

entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dashboard/DashboardInfoTemplate.java

@@ -22,6 +22,7 @@ public class DashboardInfoTemplate implements ArrayMergingTemplate<DashboardInfo
   private static final String DATASETS_FIELD_NAME = "datasets";
   private static final String CHARTS_FIELD_NAME = "charts";
   private static final String DESTINATION_URN_FIELD_NAME = "destinationUrn";
+  private static final String DASHBOARDS_FIELD_NAME = "dashboards";
 
   @Override
   public DashboardInfo getSubtype(RecordTemplate recordTemplate) throws ClassCastException {
@@ -74,6 +75,12 @@ public JsonNode transformFields(JsonNode baseNode) {
             DATASET_EDGES_FIELD_NAME,
             Collections.singletonList(DESTINATION_URN_FIELD_NAME));
 
+    transformedNode =
+        arrayFieldToMap(
+            transformedNode,
+            DASHBOARDS_FIELD_NAME,
+            Collections.singletonList(DESTINATION_URN_FIELD_NAME));
+
     transformedNode =
         arrayFieldToMap(transformedNode, DATASETS_FIELD_NAME, Collections.emptyList());
 
@@ -97,6 +104,12 @@ public JsonNode rebaseFields(JsonNode patched) {
             CHART_EDGES_FIELD_NAME,
             Collections.singletonList(DESTINATION_URN_FIELD_NAME));
 
+    rebasedNode =
+        transformedMapToArray(
+            rebasedNode,
+            DASHBOARDS_FIELD_NAME,
+            Collections.singletonList(DESTINATION_URN_FIELD_NAME));
+
     rebasedNode = transformedMapToArray(rebasedNode, DATASETS_FIELD_NAME, Collections.emptyList());
     rebasedNode = transformedMapToArray(rebasedNode, CHARTS_FIELD_NAME, Collections.emptyList());
 

entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/DashboardInfoTemplateTest.java

@@ -31,4 +31,26 @@ public void testDashboardInfoTemplate() throws Exception {
         UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)"),
         result.getDatasetEdges().get(0).getDestinationUrn());
   }
+
+  @Test
+  public void testDashboardInfoTemplateDashboardsField() throws Exception {
+    DashboardInfoTemplate dashboardInfoTemplate = new DashboardInfoTemplate();
+    DashboardInfo dashboardInfo = dashboardInfoTemplate.getDefault();
+    JsonPatchBuilder jsonPatchBuilder = Json.createPatchBuilder();
+    jsonPatchBuilder.add(
+        "/dashboards/urn:li:dashboard:(urn:li:dataPlatform:tableau,SampleDashboard,PROD)",
+        Json.createObjectBuilder()
+            .add(
+                "destinationUrn",
+                Json.createValue(
+                    "urn:li:dashboard:(urn:li:dataPlatform:tableau,SampleDashboard,PROD)"))
+            .build());
+
+    DashboardInfo result =
+        dashboardInfoTemplate.applyPatch(dashboardInfo, jsonPatchBuilder.build());
+
+    Assert.assertEquals(
+        UrnUtils.getUrn("urn:li:dashboard:(urn:li:dataPlatform:tableau,SampleDashboard,PROD)"),
+        result.getDashboards().get(0).getDestinationUrn());
+  }
 }

metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py

@@ -102,6 +102,10 @@ def convert_dashboard_info_to_patch(
     if aspect.datasets:
         patch_builder.add_datasets(aspect.datasets)
 
+    if aspect.dashboards:
+        for dashboard in aspect.dashboards:
+            patch_builder.add_dashboard(dashboard)
+
     if aspect.access:
         patch_builder.set_access(aspect.access)
 

metadata-ingestion/src/datahub/specific/dashboard.py

@@ -161,7 +161,7 @@ def add_chart_edge(self, chart: Union[Edge, Urn, str]) -> "DashboardPatchBuilder
                 lastModified=self._mint_auditstamp(),
             )
 
-        self._ensure_urn_type("dataset", [chart_edge], "add_chart_edge")
+        self._ensure_urn_type("chart", [chart_edge], "add_chart_edge")
         self._add_patch(
             DashboardInfo.ASPECT_NAME,
             "add",
@@ -271,6 +271,48 @@ def add_datasets(
 
         return self
 
+    def add_dashboard(
+        self, dashboard: Union[Edge, Urn, str]
+    ) -> "DashboardPatchBuilder":
+        """
+        Adds an dashboard to the DashboardPatchBuilder.
+
+        Args:
+            dashboard: The dashboard, which can be an Edge object, Urn object, or a string.
+
+        Returns:
+            The DashboardPatchBuilder instance.
+
+        Raises:
+            ValueError: If the dashboard is not a Dashboard urn.
+
+        Notes:
+            If `dashboard` is an Edge object, it is used directly. If `dashboard` is a Urn object or string,
+            it is converted to an Edge object and added with default audit stamps.
+        """
+        if isinstance(dashboard, Edge):
+            dashboard_urn: str = dashboard.destinationUrn
+            dashboard_edge: Edge = dashboard
+        elif isinstance(dashboard, (Urn, str)):
+            dashboard_urn = str(dashboard)
+            if not dashboard_urn.startswith("urn:li:dashboard:"):
+                raise ValueError(f"Input {dashboard} is not a Dashboard urn")
+
+            dashboard_edge = Edge(
+                destinationUrn=dashboard_urn,
+                created=self._mint_auditstamp(),
+                lastModified=self._mint_auditstamp(),
+            )
+
+        self._ensure_urn_type("dashboard", [dashboard_edge], "add_dashboard")
+        self._add_patch(
+            DashboardInfo.ASPECT_NAME,
+            "add",
+            path=("dashboards", dashboard_urn),
+            value=dashboard_edge,
+        )
+        return self
+
     def set_dashboard_url(
         self, dashboard_url: Optional[str]
     ) -> "DashboardPatchBuilder":