Merge branch 'master' into openapi-ingest

Author: david-leifker

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/util/SystemMetadataUtils.java

@@ -2,10 +2,12 @@
 
 import static com.linkedin.metadata.Constants.DEFAULT_RUN_ID;
 
+import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.entity.EnvelopedAspectMap;
 import com.linkedin.mxe.SystemMetadata;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.Comparator;
+import java.util.Objects;
+import java.util.Optional;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
@@ -15,51 +17,34 @@ private SystemMetadataUtils() {}
 
   @Nullable
   public static Long getLastIngestedTime(@Nonnull EnvelopedAspectMap aspectMap) {
-    RunInfo lastIngestionRun = getLastIngestionRun(aspectMap);
-    return lastIngestionRun != null ? lastIngestionRun.getTime() : null;
+    return getLastIngestionRun(aspectMap).map(RunInfo::getTime).orElse(null);
   }
 
   @Nullable
   public static String getLastIngestedRunId(@Nonnull EnvelopedAspectMap aspectMap) {
-    RunInfo lastIngestionRun = getLastIngestionRun(aspectMap);
-    return lastIngestionRun != null ? lastIngestionRun.getId() : null;
+    return getLastIngestionRun(aspectMap).map(RunInfo::getId).orElse(null);
   }
 
   /**
-   * Returns a sorted list of all of the most recent ingestion runs based on the most recent aspects
-   * present for the entity.
+   * Returns the most recent ingestion run based on the most recent aspects present for the entity.
    */
   @Nonnull
-  public static List<RunInfo> getLastIngestionRuns(@Nonnull EnvelopedAspectMap aspectMap) {
-    final List<RunInfo> runs = new ArrayList<>();
-    for (String aspect : aspectMap.keySet()) {
-      if (aspectMap.get(aspect).hasSystemMetadata()) {
-        SystemMetadata systemMetadata = aspectMap.get(aspect).getSystemMetadata();
-        if (systemMetadata.hasLastRunId()
-            && !systemMetadata.getLastRunId().equals(DEFAULT_RUN_ID)
-            && systemMetadata.hasLastObserved()) {
-          Long lastObserved = systemMetadata.getLastObserved();
-          String runId = systemMetadata.getLastRunId();
-          RunInfo run = new RunInfo(runId, lastObserved);
-          runs.add(run);
-        } else if (systemMetadata.hasRunId()
-            && !systemMetadata.getRunId().equals(DEFAULT_RUN_ID)
-            && systemMetadata.hasLastObserved()) {
-          // Handle the legacy case: Check original run ids.
-          Long lastObserved = systemMetadata.getLastObserved();
-          String runId = systemMetadata.getRunId();
-          RunInfo run = new RunInfo(runId, lastObserved);
-          runs.add(run);
-        }
-      }
-    }
-    runs.sort((a, b) -> Long.compare(b.getTime(), a.getTime()));
-    return runs;
-  }
-
-  @Nullable
-  private static RunInfo getLastIngestionRun(@Nonnull EnvelopedAspectMap aspectMap) {
-    List<RunInfo> runs = getLastIngestionRuns(aspectMap);
-    return !runs.isEmpty() ? runs.get(0) : null; // Just take the first, to get the most recent run.
+  private static Optional<RunInfo> getLastIngestionRun(@Nonnull EnvelopedAspectMap aspectMap) {
+    return aspectMap.values().stream()
+        .filter(EnvelopedAspect::hasSystemMetadata)
+        .map(EnvelopedAspect::getSystemMetadata)
+        .filter(SystemMetadata::hasLastObserved)
+        .map(
+            systemMetadata ->
+                Optional.ofNullable(systemMetadata.getLastRunId())
+                    .filter(lastRunId -> !lastRunId.equals(DEFAULT_RUN_ID))
+                    .or(
+                        () ->
+                            Optional.ofNullable(systemMetadata.getRunId())
+                                .filter(runId -> !runId.equals(DEFAULT_RUN_ID)))
+                    .map(runId -> new RunInfo(runId, systemMetadata.getLastObserved()))
+                    .orElse(null))
+        .filter(Objects::nonNull)
+        .max(Comparator.comparingLong(RunInfo::getTime));
   }
 }

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/SystemMetadataUtilsTest.java

@@ -3,12 +3,10 @@
 import static com.linkedin.metadata.Constants.DEFAULT_RUN_ID;
 import static org.testng.Assert.*;
 
-import com.linkedin.datahub.graphql.types.common.mappers.util.RunInfo;
 import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
 import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.entity.EnvelopedAspectMap;
 import com.linkedin.mxe.SystemMetadata;
-import java.util.List;
 import org.testng.annotations.Test;
 
 public class SystemMetadataUtilsTest {
@@ -63,32 +61,6 @@ public void testGetLastIngestedRunId() {
     assertEquals(lastRunId, "real-id-1");
   }
 
-  @Test
-  public void testGetLastIngestedRuns() {
-    EnvelopedAspectMap aspectMap = new EnvelopedAspectMap();
-    aspectMap.put(
-        "default-run-id",
-        new EnvelopedAspect()
-            .setSystemMetadata(
-                new SystemMetadata().setRunId(DEFAULT_RUN_ID).setLastObserved(recentLastObserved)));
-    aspectMap.put(
-        "real-run-id",
-        new EnvelopedAspect()
-            .setSystemMetadata(
-                new SystemMetadata().setRunId("real-id-1").setLastObserved(mediumLastObserved)));
-    aspectMap.put(
-        "real-run-id2",
-        new EnvelopedAspect()
-            .setSystemMetadata(
-                new SystemMetadata().setRunId("real-id-2").setLastObserved(distantLastObserved)));
-
-    List<RunInfo> runs = SystemMetadataUtils.getLastIngestionRuns(aspectMap);
-
-    assertEquals(runs.size(), 2);
-    assertEquals(runs.get(0), new RunInfo("real-id-1", mediumLastObserved));
-    assertEquals(runs.get(1), new RunInfo("real-id-2", distantLastObserved));
-  }
-
   @Test
   public void testGetLastIngestedTimeAllDefaultRunIds() {
     EnvelopedAspectMap aspectMap = new EnvelopedAspectMap();

docs/managed-datahub/release-notes/v_0_3_8.md

@@ -17,6 +17,31 @@ Recommended CLI/SDK
 ## Release Changelog
 ---
 
+### v0.3.8.2
+- Helm Chart Requirement: 1.5.0+
+- [Remote Executor]: Add support for file-based secret manager.
+- [UI]: Show user that manually triggered an ingestion.
+- [UI]: Add option to copy custom ownership type urn.
+- [Ingestion]: Add Restore Indicies as a DataHub Cloud ingestion source.
+- [Observability]: Fix ingestionSourceForEntity to only fetches sources that can execute assertions based on aspect source.
+- [SQL] Fix **No rows** updated exception.
+- [Search] Fix autocomplete duplicate field.
+- [Lineage] Fix lineage counts on search cards with ghost entities.
+- [UI] Show editable field info for fields based on exact fieldPath version.
+- [Metadata Tests] Add browse path v2 container as a selectable property.
+- [Modelling] Fix DashboardContainsDashboard relationship in DashboardInfo aspect.
+- [UI] Consider Dataset Profiles, Status and Browse Path v2 aspects when computing last ingested timestamp.
+- [Metadata Tests] Add ownership type as a filter for selecting assets.
+- [API] Disallow deleting system users.
+- [UI] Add 'Created At' filter.
+- [Search Ranking] Reset ranking on unused datasets.
+- [Integrations] Pre-install required duckdb extensions.
+- [ElasticSearch] Fix runId no document found exception.
+- [Modelling] Add UrnValidation PDL annotation.
+- [UI] Fix bug with nested selects and selectable parents
+- [UI] Downgrade V2 field paths in column stats table
+
+
 ### v0.3.8
 
 - All changes in https://github.com/datahub-project/datahub/releases/tag/v0.14.1

metadata-ingestion/developing.md

@@ -155,8 +155,8 @@ Instead, we recommend using UI-based ingestion or isolating the ingestion pipeli
 The syntax for installing plugins is slightly different in development. For example:
 
 ```diff
-- pip install 'acryl-datahub[bigquery,datahub-rest]'
-+ pip install -e '.[bigquery,datahub-rest]'
+- uv pip install 'acryl-datahub[bigquery,datahub-rest]'
++ uv pip install -e '.[bigquery,datahub-rest]'
 ```
 
 ## Architecture
@@ -180,7 +180,7 @@ The architecture of this metadata ingestion framework is heavily inspired by [Ap
 We use ruff, and mypy to ensure consistent code style and quality.
 
 ```shell
-# Assumes: pip install -e '.[dev]' and venv is activated
+# Assumes: ../gradlew :metadata-ingestion:installDev and venv is activated
 ruff check src/ tests/
 mypy src/ tests/
 ```
@@ -247,11 +247,8 @@ In order to ensure that the configs are consistent and easy to use, we have a fe
 ```shell
 # Follow standard install from source procedure - see above.
 
-# Install, including all dev requirements.
-pip install -e '.[dev]'
-
-# For running integration tests, you can use
-pip install -e '.[integration-tests]'
+# Install all dev and test requirements.
+../gradlew :metadata-ingestion:installDevTest
 
 # Run the full testing suite
 pytest -vv