Merge branch 'master' into openapi-ingest

Author: david-leifker

build.gradle

@@ -60,6 +60,7 @@ buildscript {
   ext.googleJavaFormatVersion = '1.18.1'
   ext.openLineageVersion = '1.25.0'
   ext.logbackClassicJava8 = '1.2.12'
+  ext.awsSdk2Version = '2.30.33'
 
   ext.docker_registry = 'acryldata'
 
@@ -120,12 +121,12 @@ project.ext.externalDependency = [
     'assertJ': 'org.assertj:assertj-core:3.11.1',
     'avro': 'org.apache.avro:avro:1.11.4',
     'avroCompiler': 'org.apache.avro:avro-compiler:1.11.4',
-    'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.17',
-    'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:2.0.3',
-    'awsS3': 'software.amazon.awssdk:s3:2.26.21',
-    'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.13',
-    'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.2',
-    'awsRds':'software.amazon.awssdk:rds:2.18.24',
+    'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.23',
+    'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:2.3.0',
+    'awsS3': "software.amazon.awssdk:s3:$awsSdk2Version",
+    'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.15',
+    'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:2.5.4',
+    'awsRds':"software.amazon.awssdk:rds:$awsSdk2Version",
     'cacheApi': 'javax.cache:cache-api:1.1.0',
     'commonsCli': 'commons-cli:commons-cli:1.5.0',
     'commonsIo': 'commons-io:commons-io:2.17.0',
@@ -240,7 +241,7 @@ project.ext.externalDependency = [
     'playFilters': "com.typesafe.play:filters-helpers_$playScalaVersion:$playVersion",
     'pac4j': 'org.pac4j:pac4j-oidc:6.0.6',
     'playPac4j': "org.pac4j:play-pac4j_$playScalaVersion:12.0.0-PLAY2.8",
-    'postgresql': 'org.postgresql:postgresql:42.7.4',
+    'postgresql': 'org.postgresql:postgresql:42.7.5',
     'protobuf': 'com.google.protobuf:protobuf-java:3.25.5',
     'grpcProtobuf': 'io.grpc:grpc-protobuf:1.53.0',
     'rangerCommons': 'org.apache.ranger:ranger-plugins-common:2.3.0',

datahub-frontend/app/auth/GuestAuthenticationConfigs.java

@@ -0,0 +1,40 @@
+package auth;
+
+public class GuestAuthenticationConfigs {
+  public static final String GUEST_ENABLED_CONFIG_PATH = "auth.guest.enabled";
+  public static final String GUEST_USER_CONFIG_PATH = "auth.guest.user";
+  public static final String GUEST_PATH_CONFIG_PATH = "auth.guest.path";
+  public static final String DEFAULT_GUEST_USER_NAME = "guest";
+  public static final String DEFAULT_GUEST_PATH = "/public";
+
+  private Boolean isEnabled = false;
+  private String guestUser =
+      DEFAULT_GUEST_USER_NAME; // Default if not defined but guest auth is enabled.
+  private String guestPath =
+      DEFAULT_GUEST_PATH; // The path for initial access to login as guest and bypass login page.
+
+  public GuestAuthenticationConfigs(final com.typesafe.config.Config configs) {
+    if (configs.hasPath(GUEST_ENABLED_CONFIG_PATH)
+        && configs.getBoolean(GUEST_ENABLED_CONFIG_PATH)) {
+      isEnabled = true;
+    }
+    if (configs.hasPath(GUEST_USER_CONFIG_PATH)) {
+      guestUser = configs.getString(GUEST_USER_CONFIG_PATH);
+    }
+    if (configs.hasPath(GUEST_PATH_CONFIG_PATH)) {
+      guestPath = configs.getString(GUEST_PATH_CONFIG_PATH);
+    }
+  }
+
+  public boolean isGuestEnabled() {
+    return isEnabled;
+  }
+
+  public String getGuestUser() {
+    return guestUser;
+  }
+
+  public String getGuestPath() {
+    return guestPath;
+  }
+}

datahub-frontend/app/controllers/AuthenticationController.java

@@ -6,6 +6,7 @@
 
 import auth.AuthUtils;
 import auth.CookieConfigs;
+import auth.GuestAuthenticationConfigs;
 import auth.JAASConfigs;
 import auth.NativeAuthenticationConfigs;
 import auth.sso.SsoManager;
@@ -58,6 +59,8 @@ public class AuthenticationController extends Controller {
   private final CookieConfigs cookieConfigs;
   private final JAASConfigs jaasConfigs;
   private final NativeAuthenticationConfigs nativeAuthenticationConfigs;
+  private final GuestAuthenticationConfigs guestAuthenticationConfigs;
+
   private final boolean verbose;
 
   @Inject private org.pac4j.core.config.Config ssoConfig;
@@ -73,6 +76,7 @@ public AuthenticationController(@Nonnull Config configs) {
     cookieConfigs = new CookieConfigs(configs);
     jaasConfigs = new JAASConfigs(configs);
     nativeAuthenticationConfigs = new NativeAuthenticationConfigs(configs);
+    guestAuthenticationConfigs = new GuestAuthenticationConfigs(configs);
     verbose = configs.hasPath(AUTH_VERBOSE_LOGGING) && configs.getBoolean(AUTH_VERBOSE_LOGGING);
   }
 
@@ -110,6 +114,23 @@ public Result authenticate(Http.Request request) {
       return Results.redirect(redirectPath);
     }
 
+    if (guestAuthenticationConfigs.isGuestEnabled()
+        && guestAuthenticationConfigs.getGuestPath().equals(redirectPath)) {
+      final String accessToken =
+          authClient.generateSessionTokenForUser(guestAuthenticationConfigs.getGuestUser());
+      redirectPath =
+          "/"; // We requested guest login by accessing {guestPath} URL. It is not really a target.
+      CorpuserUrn guestUserUrn = new CorpuserUrn(guestAuthenticationConfigs.getGuestUser());
+      return Results.redirect(redirectPath)
+          .withSession(createSessionMap(guestUserUrn.toString(), accessToken))
+          .withCookies(
+              createActorCookie(
+                  guestUserUrn.toString(),
+                  cookieConfigs.getTtlInHours(),
+                  cookieConfigs.getAuthCookieSameSite(),
+                  cookieConfigs.getAuthCookieSecure()));
+    }
+
     // 1. If SSO is enabled, redirect to IdP if not authenticated.
     if (ssoManager.isSsoEnabled()) {
       return redirectToIdentityProvider(request, redirectPath)

datahub-frontend/conf/application.conf

@@ -203,6 +203,11 @@ auth.oidc.grantType = ${?AUTH_OIDC_GRANT_TYPE}
 #
 auth.jaas.enabled = ${?AUTH_JAAS_ENABLED}
 auth.native.enabled = ${?AUTH_NATIVE_ENABLED}
+auth.guest.enabled = ${?GUEST_AUTHENTICATION_ENABLED}
+# The name of the guest user id
+auth.guest.user = ${?GUEST_AUTHENTICATION_USER}
+# The path to bypass login page and get logged in as guest
+auth.guest.path = ${?GUEST_AUTHENTICATION_PATH}
 
 # Enforces the usage of a valid email for user sign up
 auth.native.signUp.enforceValidEmail = true

datahub-frontend/test/security/GuestAuthenticationConfigsTest.java

@@ -0,0 +1,61 @@
+package security;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import auth.GuestAuthenticationConfigs;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.junitpioneer.jupiter.ClearEnvironmentVariable;
+import org.junitpioneer.jupiter.SetEnvironmentVariable;
+
+@TestInstance(TestInstance.Lifecycle.PER_METHOD)
+@SetEnvironmentVariable(key = "DATAHUB_SECRET", value = "test")
+@SetEnvironmentVariable(key = "KAFKA_BOOTSTRAP_SERVER", value = "")
+@SetEnvironmentVariable(key = "DATAHUB_ANALYTICS_ENABLED", value = "false")
+@SetEnvironmentVariable(key = "AUTH_OIDC_ENABLED", value = "true")
+@SetEnvironmentVariable(key = "AUTH_OIDC_JIT_PROVISIONING_ENABLED", value = "false")
+@SetEnvironmentVariable(key = "AUTH_OIDC_CLIENT_ID", value = "testclient")
+@SetEnvironmentVariable(key = "AUTH_OIDC_CLIENT_SECRET", value = "testsecret")
+@SetEnvironmentVariable(key = "AUTH_VERBOSE_LOGGING", value = "true")
+class GuestAuthenticationConfigsTest {
+
+  @BeforeEach
+  @ClearEnvironmentVariable(key = "GUEST_AUTHENTICATION_ENABLED")
+  @ClearEnvironmentVariable(key = "GUEST_AUTHENTICATION_USER")
+  @ClearEnvironmentVariable(key = "GUEST_AUTHENTICATION_PATH")
+  public void clearConfigCache() {
+    ConfigFactory.invalidateCaches();
+  }
+
+  @Test
+  public void testGuestConfigDisabled() {
+    Config config = ConfigFactory.load();
+    GuestAuthenticationConfigs guestAuthConfig = new GuestAuthenticationConfigs(config);
+    assertFalse(guestAuthConfig.isGuestEnabled());
+  }
+
+  @Test
+  @SetEnvironmentVariable(key = "GUEST_AUTHENTICATION_ENABLED", value = "true")
+  public void testGuestConfigEnabled() {
+    Config config = ConfigFactory.load();
+    GuestAuthenticationConfigs guestAuthConfig = new GuestAuthenticationConfigs(config);
+    assertTrue(guestAuthConfig.isGuestEnabled());
+    assertEquals("guest", guestAuthConfig.getGuestUser());
+    assertEquals("/public", guestAuthConfig.getGuestPath());
+  }
+
+  @Test
+  @SetEnvironmentVariable(key = "GUEST_AUTHENTICATION_ENABLED", value = "true")
+  @SetEnvironmentVariable(key = "GUEST_AUTHENTICATION_USER", value = "publicUser")
+  @SetEnvironmentVariable(key = "GUEST_AUTHENTICATION_PATH", value = "/publicPath")
+  public void testGuestConfigWithUserEnabled() {
+    Config config = ConfigFactory.load();
+    GuestAuthenticationConfigs guestAuthConfig = new GuestAuthenticationConfigs(config);
+    assertTrue(guestAuthConfig.isGuestEnabled());
+    assertEquals("publicUser", guestAuthConfig.getGuestUser());
+    assertEquals("/publicPath", guestAuthConfig.getGuestPath());
+  }
+}

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -1026,7 +1026,8 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) {
                     new ScrollAcrossEntitiesResolver(this.entityClient, this.viewService))
                 .dataFetcher(
                     "searchAcrossLineage",
-                    new SearchAcrossLineageResolver(this.entityClient, this.entityRegistry))
+                    new SearchAcrossLineageResolver(
+                        this.entityClient, this.entityRegistry, this.viewService))
                 .dataFetcher(
                     "scrollAcrossLineage", new ScrollAcrossLineageResolver(this.entityClient))
                 .dataFetcher(

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java

@@ -204,6 +204,8 @@ public CompletableFuture<AppConfig> get(final DataFetchingEnvironment environmen
             .setShowNavBarRedesign(_featureFlags.isShowNavBarRedesign())
             .setShowAutoCompleteResults(_featureFlags.isShowAutoCompleteResults())
             .setEntityVersioningEnabled(_featureFlags.isEntityVersioning())
+            .setShowSearchBarAutocompleteRedesign(
+                _featureFlags.isShowSearchBarAutocompleteRedesign())
             .build();
 
     appConfig.setFeatureFlags(featureFlagsConfig);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java

@@ -6,6 +6,7 @@
 
 import com.google.common.collect.ImmutableSet;
 import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils;
 import com.linkedin.datahub.graphql.generated.AndFilterInput;
@@ -25,8 +26,12 @@
 import com.linkedin.metadata.query.LineageFlags;
 import com.linkedin.metadata.query.SearchFlags;
 import com.linkedin.metadata.query.filter.Filter;
+import com.linkedin.metadata.query.filter.SortCriterion;
 import com.linkedin.metadata.search.LineageSearchResult;
+import com.linkedin.metadata.service.ViewService;
+import com.linkedin.metadata.utils.elasticsearch.FilterUtils;
 import com.linkedin.r2.RemoteInvocationException;
+import com.linkedin.view.DataHubViewInfo;
 import graphql.VisibleForTesting;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
@@ -53,10 +58,13 @@ public class SearchAcrossLineageResolver
 
   private final EntityRegistry _entityRegistry;
 
+  private final ViewService _viewService;
+
   @VisibleForTesting final Set<String> _allEntities;
   private final List<String> _allowedEntities;
 
-  public SearchAcrossLineageResolver(EntityClient entityClient, EntityRegistry entityRegistry) {
+  public SearchAcrossLineageResolver(
+      EntityClient entityClient, EntityRegistry entityRegistry, final ViewService viewService) {
     this._entityClient = entityClient;
     this._entityRegistry = entityRegistry;
     this._allEntities =
@@ -68,6 +76,8 @@ public SearchAcrossLineageResolver(EntityClient entityClient, EntityRegistry ent
         this._allEntities.stream()
             .filter(e -> !TRANSIENT_ENTITIES.contains(e))
             .collect(Collectors.toList());
+
+    this._viewService = viewService;
   }
 
   private List<String> getEntityNamesFromInput(List<EntityType> inputTypes) {
@@ -127,6 +137,13 @@ public CompletableFuture<SearchAcrossLineageResults> get(DataFetchingEnvironment
         com.linkedin.metadata.graph.LineageDirection.valueOf(lineageDirection.toString());
     return GraphQLConcurrencyUtils.supplyAsync(
         () -> {
+          final DataHubViewInfo maybeResolvedView =
+              (input.getViewUrn() != null)
+                  ? resolveView(
+                      context.getOperationContext(),
+                      _viewService,
+                      UrnUtils.getUrn(input.getViewUrn()))
+                  : null;
           try {
             log.debug(
                 "Executing search across relationships: source urn {}, direction {}, entity types {}, query {}, filters: {}, start: {}, count: {}",
@@ -138,8 +155,13 @@ public CompletableFuture<SearchAcrossLineageResults> get(DataFetchingEnvironment
                 start,
                 count);
 
-            final Filter filter =
+            final Filter baseFilter =
                 ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters());
+            Filter filter =
+                maybeResolvedView != null
+                    ? FilterUtils.combineFilters(
+                        baseFilter, maybeResolvedView.getDefinition().getFilter())
+                    : baseFilter;
             final SearchFlags searchFlags;
             com.linkedin.datahub.graphql.generated.SearchFlags inputFlags = input.getSearchFlags();
             if (inputFlags != null) {
@@ -150,6 +172,7 @@ public CompletableFuture<SearchAcrossLineageResults> get(DataFetchingEnvironment
             } else {
               searchFlags = new SearchFlags().setFulltext(true).setSkipHighlighting(true);
             }
+            List<SortCriterion> sortCriteria = SearchUtils.getSortCriteria(input.getSortInput());
             LineageSearchResult salResults =
                 _entityClient.searchAcrossLineage(
                     context
@@ -162,7 +185,7 @@ public CompletableFuture<SearchAcrossLineageResults> get(DataFetchingEnvironment
                     sanitizedQuery,
                     maxHops,
                     filter,
-                    null,
+                    sortCriteria,
                     start,
                     count);
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainType.java

@@ -40,7 +40,8 @@ public class DomainType
           Constants.OWNERSHIP_ASPECT_NAME,
           Constants.INSTITUTIONAL_MEMORY_ASPECT_NAME,
           Constants.STRUCTURED_PROPERTIES_ASPECT_NAME,
-          Constants.FORMS_ASPECT_NAME);
+          Constants.FORMS_ASPECT_NAME,
+          Constants.DISPLAY_PROPERTIES_ASPECT_NAME);
   private final EntityClient _entityClient;
 
   public DomainType(final EntityClient entityClient) {

datahub-graphql-core/src/main/resources/app.graphql

@@ -593,6 +593,11 @@ type FeatureFlagsConfig {
   If turned on, exposes the versioning feature by allowing users to link entities in the UI.
   """
   entityVersioningEnabled: Boolean!
+
+  """
+  If turned on, show the redesigned search bar's autocomplete
+  """
+  showSearchBarAutocompleteRedesign: Boolean!
 }
 
 """

datahub-graphql-core/src/main/resources/search.graphql

@@ -417,6 +417,16 @@ input SearchAcrossLineageInput {
   Flags controlling the lineage query
   """
   lineageFlags: LineageFlags
+
+  """
+  Optional - A View to apply when generating results
+  """
+  viewUrn: String
+
+  """
+  Optional - Information on how to sort this search result
+  """
+  sortInput: SearchSortInput
 }
 
 """

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolverTest.java

@@ -31,6 +31,7 @@
 import com.linkedin.metadata.search.LineageSearchEntityArray;
 import com.linkedin.metadata.search.MatchedFieldArray;
 import com.linkedin.metadata.search.SearchResultMetadata;
+import com.linkedin.metadata.service.ViewService;
 import graphql.schema.DataFetchingEnvironment;
 import io.datahubproject.metadata.context.OperationContext;
 import java.io.InputStream;
@@ -76,7 +77,7 @@ public void testAllEntitiesInitialization() {
     InputStream inputStream = ClassLoader.getSystemResourceAsStream("entity-registry.yml");
     EntityRegistry entityRegistry = new ConfigEntityRegistry(inputStream);
     SearchAcrossLineageResolver resolver =
-        new SearchAcrossLineageResolver(_entityClient, entityRegistry);
+        new SearchAcrossLineageResolver(_entityClient, entityRegistry, mock(ViewService.class));
     assertTrue(resolver._allEntities.contains("dataset"));
     assertTrue(resolver._allEntities.contains("dataFlow"));
     // Test for case sensitivity