Merge branch 'master' into peter/column-level-lineage

Author: PeteMango

build.gradle

@@ -60,6 +60,7 @@ buildscript {
   ext.googleJavaFormatVersion = '1.18.1'
   ext.openLineageVersion = '1.25.0'
   ext.logbackClassicJava8 = '1.2.12'
+  ext.awsSdk2Version = '2.30.33'
 
   ext.docker_registry = 'acryldata'
 
@@ -120,12 +121,12 @@ project.ext.externalDependency = [
     'assertJ': 'org.assertj:assertj-core:3.11.1',
     'avro': 'org.apache.avro:avro:1.11.4',
     'avroCompiler': 'org.apache.avro:avro-compiler:1.11.4',
-    'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.17',
-    'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:2.0.3',
-    'awsS3': 'software.amazon.awssdk:s3:2.26.21',
-    'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.13',
-    'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.2',
-    'awsRds':'software.amazon.awssdk:rds:2.18.24',
+    'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.23',
+    'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:2.3.0',
+    'awsS3': "software.amazon.awssdk:s3:$awsSdk2Version",
+    'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.15',
+    'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:2.5.4',
+    'awsRds':"software.amazon.awssdk:rds:$awsSdk2Version",
     'cacheApi': 'javax.cache:cache-api:1.1.0',
     'commonsCli': 'commons-cli:commons-cli:1.5.0',
     'commonsIo': 'commons-io:commons-io:2.17.0',
@@ -240,7 +241,7 @@ project.ext.externalDependency = [
     'playFilters': "com.typesafe.play:filters-helpers_$playScalaVersion:$playVersion",
     'pac4j': 'org.pac4j:pac4j-oidc:6.0.6',
     'playPac4j': "org.pac4j:play-pac4j_$playScalaVersion:12.0.0-PLAY2.8",
-    'postgresql': 'org.postgresql:postgresql:42.7.4',
+    'postgresql': 'org.postgresql:postgresql:42.7.5',
     'protobuf': 'com.google.protobuf:protobuf-java:3.25.5',
     'grpcProtobuf': 'io.grpc:grpc-protobuf:1.53.0',
     'rangerCommons': 'org.apache.ranger:ranger-plugins-common:2.3.0',

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/util/SystemMetadataUtils.java

@@ -2,10 +2,12 @@
 
 import static com.linkedin.metadata.Constants.DEFAULT_RUN_ID;
 
+import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.entity.EnvelopedAspectMap;
 import com.linkedin.mxe.SystemMetadata;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.Comparator;
+import java.util.Objects;
+import java.util.Optional;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
@@ -15,51 +17,34 @@ private SystemMetadataUtils() {}
 
   @Nullable
   public static Long getLastIngestedTime(@Nonnull EnvelopedAspectMap aspectMap) {
-    RunInfo lastIngestionRun = getLastIngestionRun(aspectMap);
-    return lastIngestionRun != null ? lastIngestionRun.getTime() : null;
+    return getLastIngestionRun(aspectMap).map(RunInfo::getTime).orElse(null);
   }
 
   @Nullable
   public static String getLastIngestedRunId(@Nonnull EnvelopedAspectMap aspectMap) {
-    RunInfo lastIngestionRun = getLastIngestionRun(aspectMap);
-    return lastIngestionRun != null ? lastIngestionRun.getId() : null;
+    return getLastIngestionRun(aspectMap).map(RunInfo::getId).orElse(null);
   }
 
   /**
-   * Returns a sorted list of all of the most recent ingestion runs based on the most recent aspects
-   * present for the entity.
+   * Returns the most recent ingestion run based on the most recent aspects present for the entity.
    */
   @Nonnull
-  public static List<RunInfo> getLastIngestionRuns(@Nonnull EnvelopedAspectMap aspectMap) {
-    final List<RunInfo> runs = new ArrayList<>();
-    for (String aspect : aspectMap.keySet()) {
-      if (aspectMap.get(aspect).hasSystemMetadata()) {
-        SystemMetadata systemMetadata = aspectMap.get(aspect).getSystemMetadata();
-        if (systemMetadata.hasLastRunId()
-            && !systemMetadata.getLastRunId().equals(DEFAULT_RUN_ID)
-            && systemMetadata.hasLastObserved()) {
-          Long lastObserved = systemMetadata.getLastObserved();
-          String runId = systemMetadata.getLastRunId();
-          RunInfo run = new RunInfo(runId, lastObserved);
-          runs.add(run);
-        } else if (systemMetadata.hasRunId()
-            && !systemMetadata.getRunId().equals(DEFAULT_RUN_ID)
-            && systemMetadata.hasLastObserved()) {
-          // Handle the legacy case: Check original run ids.
-          Long lastObserved = systemMetadata.getLastObserved();
-          String runId = systemMetadata.getRunId();
-          RunInfo run = new RunInfo(runId, lastObserved);
-          runs.add(run);
-        }
-      }
-    }
-    runs.sort((a, b) -> Long.compare(b.getTime(), a.getTime()));
-    return runs;
-  }
-
-  @Nullable
-  private static RunInfo getLastIngestionRun(@Nonnull EnvelopedAspectMap aspectMap) {
-    List<RunInfo> runs = getLastIngestionRuns(aspectMap);
-    return !runs.isEmpty() ? runs.get(0) : null; // Just take the first, to get the most recent run.
+  private static Optional<RunInfo> getLastIngestionRun(@Nonnull EnvelopedAspectMap aspectMap) {
+    return aspectMap.values().stream()
+        .filter(EnvelopedAspect::hasSystemMetadata)
+        .map(EnvelopedAspect::getSystemMetadata)
+        .filter(SystemMetadata::hasLastObserved)
+        .map(
+            systemMetadata ->
+                Optional.ofNullable(systemMetadata.getLastRunId())
+                    .filter(lastRunId -> !lastRunId.equals(DEFAULT_RUN_ID))
+                    .or(
+                        () ->
+                            Optional.ofNullable(systemMetadata.getRunId())
+                                .filter(runId -> !runId.equals(DEFAULT_RUN_ID)))
+                    .map(runId -> new RunInfo(runId, systemMetadata.getLastObserved()))
+                    .orElse(null))
+        .filter(Objects::nonNull)
+        .max(Comparator.comparingLong(RunInfo::getTime));
   }
 }

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/SystemMetadataUtilsTest.java

@@ -3,12 +3,10 @@
 import static com.linkedin.metadata.Constants.DEFAULT_RUN_ID;
 import static org.testng.Assert.*;
 
-import com.linkedin.datahub.graphql.types.common.mappers.util.RunInfo;
 import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
 import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.entity.EnvelopedAspectMap;
 import com.linkedin.mxe.SystemMetadata;
-import java.util.List;
 import org.testng.annotations.Test;
 
 public class SystemMetadataUtilsTest {
@@ -63,32 +61,6 @@ public void testGetLastIngestedRunId() {
     assertEquals(lastRunId, "real-id-1");
   }
 
-  @Test
-  public void testGetLastIngestedRuns() {
-    EnvelopedAspectMap aspectMap = new EnvelopedAspectMap();
-    aspectMap.put(
-        "default-run-id",
-        new EnvelopedAspect()
-            .setSystemMetadata(
-                new SystemMetadata().setRunId(DEFAULT_RUN_ID).setLastObserved(recentLastObserved)));
-    aspectMap.put(
-        "real-run-id",
-        new EnvelopedAspect()
-            .setSystemMetadata(
-                new SystemMetadata().setRunId("real-id-1").setLastObserved(mediumLastObserved)));
-    aspectMap.put(
-        "real-run-id2",
-        new EnvelopedAspect()
-            .setSystemMetadata(
-                new SystemMetadata().setRunId("real-id-2").setLastObserved(distantLastObserved)));
-
-    List<RunInfo> runs = SystemMetadataUtils.getLastIngestionRuns(aspectMap);
-
-    assertEquals(runs.size(), 2);
-    assertEquals(runs.get(0), new RunInfo("real-id-1", mediumLastObserved));
-    assertEquals(runs.get(1), new RunInfo("real-id-2", distantLastObserved));
-  }
-
   @Test
   public void testGetLastIngestedTimeAllDefaultRunIds() {
     EnvelopedAspectMap aspectMap = new EnvelopedAspectMap();

datahub-web-react/src/app/previewV2/BrowsePaths.tsx

@@ -71,12 +71,16 @@ const BrowsePathSection = ({ path, linksDisabled }: { path: BrowsePathEntry } &
     if (!path.entity) {
         return <PlatFormTitle>{path.name}</PlatFormTitle>;
     }
+
+    // Till we have a DataPlatform instance page
+    const hasDataPlatformInstance = path.name?.includes('dataPlatformInstance');
+
     return (
         <ContextPathEntityLink
             key={path?.entity?.urn}
             entity={path?.entity}
             style={{ fontSize: '12px' }}
-            linkDisabled={linksDisabled}
+            linkDisabled={linksDisabled || hasDataPlatformInstance}
         />
     );
 };

datahub-web-react/src/app/previewV2/ContextPath.tsx

@@ -82,6 +82,7 @@ const PlatFormTitle = styled.span`
 interface Props {
     // eslint-disable-next-line react/no-unused-prop-types
     entityLogoComponent?: JSX.Element;
+    // eslint-disable-next-line react/no-unused-prop-types
     instanceId?: string;
     // eslint-disable-next-line react/no-unused-prop-types
     typeIcon?: JSX.Element;
@@ -103,7 +104,6 @@ function ContextPath(props: Props) {
         entityType,
         parentEntities,
         browsePaths,
-        instanceId,
         entityTitleWidth = 200,
         previewType,
         isCompactView,
@@ -118,12 +118,10 @@ function ContextPath(props: Props) {
 
     const divider = <PlatformDivider>|</PlatformDivider>;
 
-    const hasPlatformInstance = !!instanceId;
     const hasBrowsePath = !!browsePaths?.path?.length && !isDefaultBrowsePath(browsePaths);
     const hasParentEntities = !!parentEntities?.length;
 
-    const showInstanceIdDivider = hasBrowsePath || hasParentEntities;
-    const showEntityTypeDivider = hasPlatformInstance || hasBrowsePath || hasParentEntities;
+    const showEntityTypeDivider = hasBrowsePath || hasParentEntities;
 
     return (
         <PlatformContentWrapper>
@@ -136,12 +134,6 @@ function ContextPath(props: Props) {
                 <PlatFormTitle>{capitalizeFirstLetterOnly(type)}</PlatFormTitle>
                 {showEntityTypeDivider && divider}
             </PlatformText>
-            {instanceId && (
-                <PlatformText>
-                    {instanceId}
-                    {showInstanceIdDivider && divider}
-                </PlatformText>
-            )}
             {hasBrowsePath ? (
                 <BrowsePaths
                     browsePaths={browsePaths}

datahub-web-react/src/app/previewV2/ContextPathEntityLink.tsx

@@ -45,7 +45,7 @@ const StyledLink = styled(Link)<{ $disabled?: boolean }>`
 
     :hover {
         color: ${({ $disabled }) => ($disabled ? REDESIGN_COLORS.LINK_GREY : colors.violet[500])};
-
+        cursor: ${({ $disabled }) => ($disabled ? 'default' : 'pointer')};
         && svg {
             color: ${({ $disabled }) => ($disabled ? REDESIGN_COLORS.LINK_GREY : colors.violet[500])};
         }

docs/how/updating-datahub.md

@@ -30,6 +30,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 - #12716: Fix the `platform_instance` being added twice to the URN. If you want to have the previous behavior back, you need to add your platform_instance twice (i.e. `plat.plat`).
 
+- #12797: Previously endpoints when used in ASYNC mode would not validate URNs, entity & aspect names immediately. Starting with this release, even in ASYNC mode, these requests will be returned with http code 400.
+
 
 ### Known Issues
 

docs/managed-datahub/release-notes/v_0_3_8.md

@@ -17,6 +17,31 @@ Recommended CLI/SDK
 ## Release Changelog
 ---
 
+### v0.3.8.2
+- Helm Chart Requirement: 1.5.0+
+- [Remote Executor]: Add support for file-based secret manager.
+- [UI]: Show user that manually triggered an ingestion.
+- [UI]: Add option to copy custom ownership type urn.
+- [Ingestion]: Add Restore Indicies as a DataHub Cloud ingestion source.
+- [Observability]: Fix ingestionSourceForEntity to only fetches sources that can execute assertions based on aspect source.
+- [SQL] Fix **No rows** updated exception.
+- [Search] Fix autocomplete duplicate field.
+- [Lineage] Fix lineage counts on search cards with ghost entities.
+- [UI] Show editable field info for fields based on exact fieldPath version.
+- [Metadata Tests] Add browse path v2 container as a selectable property.
+- [Modelling] Fix DashboardContainsDashboard relationship in DashboardInfo aspect.
+- [UI] Consider Dataset Profiles, Status and Browse Path v2 aspects when computing last ingested timestamp.
+- [Metadata Tests] Add ownership type as a filter for selecting assets.
+- [API] Disallow deleting system users.
+- [UI] Add 'Created At' filter.
+- [Search Ranking] Reset ranking on unused datasets.
+- [Integrations] Pre-install required duckdb extensions.
+- [ElasticSearch] Fix runId no document found exception.
+- [Modelling] Add UrnValidation PDL annotation.
+- [UI] Fix bug with nested selects and selectable parents
+- [UI] Downgrade V2 field paths in column stats table
+
+
 ### v0.3.8
 
 - All changes in https://github.com/datahub-project/datahub/releases/tag/v0.14.1

entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/MCPItem.java

@@ -1,9 +1,11 @@
 package com.linkedin.metadata.aspect.batch;
 
 import com.google.common.collect.ImmutableSet;
+import com.linkedin.common.urn.Urn;
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine;
 import com.linkedin.metadata.models.AspectSpec;
+import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.mxe.SystemMetadata;
 import java.util.Collections;
@@ -82,4 +84,6 @@ static boolean supportsPatch(AspectSpec aspectSpec) {
     }
     return true;
   }
+
+  default void validate(Urn urn, String aspectName, EntityRegistry entityRegistry) {}
 }

metadata-ingestion/developing.md

@@ -155,8 +155,8 @@ Instead, we recommend using UI-based ingestion or isolating the ingestion pipeli
 The syntax for installing plugins is slightly different in development. For example:
 
 ```diff
-- pip install 'acryl-datahub[bigquery,datahub-rest]'
-+ pip install -e '.[bigquery,datahub-rest]'
+- uv pip install 'acryl-datahub[bigquery,datahub-rest]'
++ uv pip install -e '.[bigquery,datahub-rest]'
 ```
 
 ## Architecture
@@ -180,7 +180,7 @@ The architecture of this metadata ingestion framework is heavily inspired by [Ap
 We use ruff, and mypy to ensure consistent code style and quality.
 
 ```shell
-# Assumes: pip install -e '.[dev]' and venv is activated
+# Assumes: ../gradlew :metadata-ingestion:installDev and venv is activated
 ruff check src/ tests/
 mypy src/ tests/
 ```
@@ -247,11 +247,8 @@ In order to ensure that the configs are consistent and easy to use, we have a fe
 ```shell
 # Follow standard install from source procedure - see above.
 
-# Install, including all dev requirements.
-pip install -e '.[dev]'
-
-# For running integration tests, you can use
-pip install -e '.[integration-tests]'
+# Install all dev and test requirements.
+../gradlew :metadata-ingestion:installDevTest
 
 # Run the full testing suite
 pytest -vv