feat(auth): user.props authentication

* document breaking change to user.props
* add flag for legacy behavior

Author: david-leifker

datahub-frontend/app/auth/AuthModule.java

@@ -181,7 +181,12 @@ protected OperationContext provideOperationContext(
       final Authentication systemAuthentication,
       final ConfigurationProvider configurationProvider) {
     ActorContext systemActorContext =
-        ActorContext.builder().systemAuth(true).authentication(systemAuthentication).build();
+        ActorContext.builder()
+            .systemAuth(true)
+            .authentication(systemAuthentication)
+            .enforceExistenceEnabled(
+                configurationProvider.getAuthentication().isEnforceExistenceEnabled())
+            .build();
     OperationContextConfig systemConfig =
         OperationContextConfig.builder()
             .viewAuthorizationConfiguration(configurationProvider.getAuthorization().getView())
@@ -197,7 +202,9 @@ protected OperationContext provideOperationContext(
         .entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY))
         .validationContext(ValidationContext.builder().alternateValidation(false).build())
         .retrieverContext(RetrieverContext.EMPTY)
-        .build(systemAuthentication);
+        .build(
+            systemAuthentication,
+            configurationProvider.getAuthentication().isEnforceExistenceEnabled());
   }
 
   @Provides

datahub-frontend/app/config/ConfigurationProvider.java

@@ -1,5 +1,6 @@
 package config;
 
+import com.datahub.authentication.AuthenticationConfiguration;
 import com.datahub.authorization.AuthorizationConfiguration;
 import com.linkedin.metadata.config.VisualConfiguration;
 import com.linkedin.metadata.config.cache.CacheConfiguration;
@@ -30,4 +31,7 @@ public class ConfigurationProvider {
 
   /** Configuration for authorization */
   private AuthorizationConfiguration authorization;
+
+  /** Configuration for authentication */
+  private AuthenticationConfiguration authentication;
 }

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java

@@ -194,7 +194,8 @@ protected OperationContext javaSystemOperationContext(
             ValidationContext.builder()
                 .alternateValidation(
                     configurationProvider.getFeatureFlags().isAlternateMCPValidation())
-                .build());
+                .build(),
+            true);
 
     entityServiceAspectRetriever.setSystemOperationContext(systemOperationContext);
     systemGraphRetriever.setSystemOperationContext(systemOperationContext);

docs/authentication/guides/add-users.md

@@ -1,3 +1,6 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
 # Onboarding Users to DataHub
 
 New user accounts can be provisioned on DataHub in 3 ways:
@@ -94,6 +97,11 @@ using this mechanism. It is highly recommended that admins change or remove the
 
 ## Adding new users using a user.props file
 
+:::NOTE
+Adding users via the `user.props` will require disabling existence checks on GMS using the `METADATA_SERVICE_AUTH_ENFORCE_EXISTENCE_ENABLED=false` environment variable or using the API to enable the user prior to login.
+The directions below demonstrate using the API to enable the user.
+:::
+
 To define a set of username / password combinations that should be allowed to log in to DataHub (in addition to the root 'datahub' user),
 create a new file called `user.props` at the file path `${HOME}/.datahub/plugins/frontend/auth/user.props` within the `datahub-frontend-react` container
 or pod.
@@ -107,6 +115,28 @@ janesmith:janespassword
 johndoe:johnspassword
 ```
 
+In order to enable the user access with the credential defined in `user.props`, set the `status` aspect on the user with an Admin user. This can be done using an API call or via the [OpenAPI UI interface](/docs/api/openapi/openapi-usage-guide.md).
+
+<Tabs>
+<TabItem value="openapi" label="OpenAPI" default>
+
+Example enabling login for the `janesmith` user from the example above. Make sure to update the example with your access token.
+
+```shell
+curl -X 'POST' \
+  'http://localhost:9002/openapi/v3/entity/corpuser/urn%3Ali%3Acorpuser%3Ajanesmith/status?async=false&systemMetadata=false&createIfEntityNotExists=false&createIfNotExists=true' \
+  -H 'accept: application/json' \
+  -H 'Content-Type: application/json' \
+  -H 'Authorization: Bearer <access token>' \
+  -d '{
+  "value": {
+    "removed": false
+  }
+}'
+```
+</TabItem>
+</Tabs>
+
 Once you've saved the file, simply start the DataHub containers & navigate to `http://localhost:9002/login`
 to verify that your new credentials work.
 

docs/how/updating-datahub.md

@@ -66,6 +66,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
   changed to NOT fill out `created` and `lastModified` auditstamps by default
   for input and output dataset edges. This should not have any user-observable
   impact (time-based lineage viz will still continue working based on observed time), but could break assumptions previously being made by clients.
+- #12158 - Users provisioned with `user.props` will need to be enabled before login in order to be granted access to DataHub.
 
 ### Potential Downtime
 

metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java

@@ -137,7 +137,7 @@ protected OperationContext sampleDataOperationContext(
 
     return testOpContext.toBuilder()
         .searchContext(SearchContext.builder().indexConvention(indexConvention).build())
-        .build(testOpContext.getSessionAuthentication());
+        .build(testOpContext.getSessionAuthentication(), true);
   }
 
   @Bean(name = "longTailOperationContext")
@@ -148,7 +148,7 @@ protected OperationContext longTailOperationContext(
 
     return testOpContext.toBuilder()
         .searchContext(SearchContext.builder().indexConvention(indexConvention).build())
-        .build(testOpContext.getSessionAuthentication());
+        .build(testOpContext.getSessionAuthentication(), true);
   }
 
   protected EntityIndexBuilders entityIndexBuildersHelper(OperationContext opContext) {

metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java

@@ -162,7 +162,7 @@ protected OperationContext searchLineageOperationContext(
 
     return testOpContext.toBuilder()
         .searchContext(SearchContext.builder().indexConvention(indexConvention).build())
-        .build(testOpContext.getSessionAuthentication());
+        .build(testOpContext.getSessionAuthentication(), true);
   }
 
   @Bean(name = "searchLineageESIndexBuilder")

metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringCommonTestConfiguration.java

@@ -95,7 +95,8 @@ public OperationContext operationContext(
         mock(ServicesRegistryContext.class),
         indexConvention,
         TestOperationContexts.emptyActiveUsersRetrieverContext(() -> entityRegistry),
-        mock(ValidationContext.class));
+        mock(ValidationContext.class),
+        true);
   }
 
   @MockBean SpringStandardPluginConfiguration springStandardPluginConfiguration;

metadata-operation-context/src/main/java/io/datahubproject/metadata/context/ActorContext.java

@@ -29,23 +29,31 @@
 @EqualsAndHashCode
 public class ActorContext implements ContextInterface {
 
-  public static ActorContext asSystem(Authentication systemAuthentication) {
-    return ActorContext.builder().systemAuth(true).authentication(systemAuthentication).build();
+  public static ActorContext asSystem(
+      Authentication systemAuthentication, boolean enforceExistenceEnabled) {
+    return ActorContext.builder()
+        .systemAuth(true)
+        .authentication(systemAuthentication)
+        .enforceExistenceEnabled(enforceExistenceEnabled)
+        .build();
   }
 
   public static ActorContext asSessionRestricted(
       Authentication authentication,
       Set<DataHubPolicyInfo> dataHubPolicySet,
-      Collection<Urn> groupMembership) {
+      Collection<Urn> groupMembership,
+      boolean enforceExistenceEnabled) {
     return ActorContext.builder()
         .systemAuth(false)
         .authentication(authentication)
         .policyInfoSet(dataHubPolicySet)
         .groupMembership(groupMembership)
+        .enforceExistenceEnabled(enforceExistenceEnabled)
         .build();
   }
 
   private final Authentication authentication;
+  private final boolean enforceExistenceEnabled;
 
   @EqualsAndHashCode.Exclude @Builder.Default
   private final Set<DataHubPolicyInfo> policyInfoSet = Collections.emptySet();
@@ -79,7 +87,7 @@ public boolean isActive(AspectRetriever aspectRetriever) {
 
     Map<String, Aspect> aspectMap = urnAspectMap.getOrDefault(selfUrn, Map.of());
 
-    if (!aspectMap.containsKey(CORP_USER_KEY_ASPECT_NAME)) {
+    if (enforceExistenceEnabled && !aspectMap.containsKey(CORP_USER_KEY_ASPECT_NAME)) {
       // user is hard deleted
       return false;
     }

metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java

@@ -152,7 +152,8 @@ public static OperationContext asSystem(
       @Nullable ServicesRegistryContext servicesRegistryContext,
       @Nullable IndexConvention indexConvention,
       @Nullable RetrieverContext retrieverContext,
-      @Nonnull ValidationContext validationContext) {
+      @Nonnull ValidationContext validationContext,
+      boolean enforceExistenceEnabled) {
     return asSystem(
         config,
         systemAuthentication,
@@ -161,7 +162,8 @@ public static OperationContext asSystem(
         indexConvention,
         retrieverContext,
         validationContext,
-        ObjectMapperContext.DEFAULT);
+        ObjectMapperContext.DEFAULT,
+        enforceExistenceEnabled);
   }
 
   public static OperationContext asSystem(
@@ -172,10 +174,15 @@ public static OperationContext asSystem(
       @Nullable IndexConvention indexConvention,
       @Nullable RetrieverContext retrieverContext,
       @Nonnull ValidationContext validationContext,
-      @Nonnull ObjectMapperContext objectMapperContext) {
+      @Nonnull ObjectMapperContext objectMapperContext,
+      boolean enforceExistenceEnabled) {
 
     ActorContext systemActorContext =
-        ActorContext.builder().systemAuth(true).authentication(systemAuthentication).build();
+        ActorContext.builder()
+            .systemAuth(true)
+            .authentication(systemAuthentication)
+            .enforceExistenceEnabled(enforceExistenceEnabled)
+            .build();
     OperationContextConfig systemConfig =
         config.toBuilder().allowSystemAuthentication(true).build();
     SearchContext systemSearchContext =
@@ -457,13 +464,16 @@ public int hashCode() {
   public static class OperationContextBuilder {
 
     @Nonnull
-    public OperationContext build(@Nonnull Authentication sessionAuthentication) {
-      return build(sessionAuthentication, false);
+    public OperationContext build(
+        @Nonnull Authentication sessionAuthentication, boolean enforceExistenceEnabled) {
+      return build(sessionAuthentication, false, enforceExistenceEnabled);
     }
 
     @Nonnull
     public OperationContext build(
-        @Nonnull Authentication sessionAuthentication, boolean skipCache) {
+        @Nonnull Authentication sessionAuthentication,
+        boolean skipCache,
+        boolean enforceExistenceEnabled) {
       final Urn actorUrn = UrnUtils.getUrn(sessionAuthentication.getActor().toUrnStr());
       final ActorContext sessionActor =
           ActorContext.builder()
@@ -476,6 +486,7 @@ public OperationContext build(
                           .equals(sessionAuthentication.getActor()))
               .policyInfoSet(this.authorizationContext.getAuthorizer().getActorPolicies(actorUrn))
               .groupMembership(this.authorizationContext.getAuthorizer().getActorGroups(actorUrn))
+              .enforceExistenceEnabled(enforceExistenceEnabled)
               .build();
       return build(sessionActor, skipCache);
     }

metadata-operation-context/src/main/java/io/datahubproject/test/metadata/context/TestOperationContexts.java

@@ -260,7 +260,8 @@ public static OperationContext systemContext(
             servicesRegistryContext,
             indexConvention,
             retrieverContext,
-            validationContext);
+            validationContext,
+            true);
 
     if (postConstruct != null) {
       postConstruct.accept(operationContext);

metadata-operation-context/src/test/java/io/datahubproject/metadata/context/ActorContextTest.java

@@ -87,42 +87,43 @@ public void actorContextId() {
     Authentication userAuth = new Authentication(new Actor(ActorType.USER, "USER"), "");
 
     assertEquals(
-        ActorContext.asSessionRestricted(userAuth, Set.of(), Set.of()).getCacheKeyComponent(),
-        ActorContext.asSessionRestricted(userAuth, Set.of(), Set.of()).getCacheKeyComponent(),
+        ActorContext.asSessionRestricted(userAuth, Set.of(), Set.of(), true).getCacheKeyComponent(),
+        ActorContext.asSessionRestricted(userAuth, Set.of(), Set.of(), true).getCacheKeyComponent(),
         "Expected equality across instances");
 
     assertEquals(
-        ActorContext.asSessionRestricted(userAuth, Set.of(), Set.of()).getCacheKeyComponent(),
+        ActorContext.asSessionRestricted(userAuth, Set.of(), Set.of(), true).getCacheKeyComponent(),
         ActorContext.asSessionRestricted(
-                userAuth, Set.of(), Set.of(UrnUtils.getUrn("urn:li:corpGroup:group1")))
+                userAuth, Set.of(), Set.of(UrnUtils.getUrn("urn:li:corpGroup:group1")), true)
             .getCacheKeyComponent(),
         "Expected no impact to cache context from group membership");
 
     assertEquals(
-        ActorContext.asSessionRestricted(userAuth, Set.of(POLICY_ABC, POLICY_D), Set.of())
+        ActorContext.asSessionRestricted(userAuth, Set.of(POLICY_ABC, POLICY_D), Set.of(), true)
             .getCacheKeyComponent(),
-        ActorContext.asSessionRestricted(userAuth, Set.of(POLICY_ABC, POLICY_D), Set.of())
+        ActorContext.asSessionRestricted(userAuth, Set.of(POLICY_ABC, POLICY_D), Set.of(), true)
             .getCacheKeyComponent(),
         "Expected equality when non-ownership policies are identical");
 
     assertNotEquals(
-        ActorContext.asSessionRestricted(userAuth, Set.of(POLICY_ABC_RESOURCE, POLICY_D), Set.of())
+        ActorContext.asSessionRestricted(
+                userAuth, Set.of(POLICY_ABC_RESOURCE, POLICY_D), Set.of(), true)
             .getCacheKeyComponent(),
-        ActorContext.asSessionRestricted(userAuth, Set.of(POLICY_ABC, POLICY_D), Set.of())
+        ActorContext.asSessionRestricted(userAuth, Set.of(POLICY_ABC, POLICY_D), Set.of(), true)
             .getCacheKeyComponent(),
         "Expected differences with non-identical resource policy");
 
     assertNotEquals(
-        ActorContext.asSessionRestricted(userAuth, Set.of(POLICY_D_OWNER), Set.of())
+        ActorContext.asSessionRestricted(userAuth, Set.of(POLICY_D_OWNER), Set.of(), true)
             .getCacheKeyComponent(),
-        ActorContext.asSessionRestricted(userAuth, Set.of(POLICY_D), Set.of())
+        ActorContext.asSessionRestricted(userAuth, Set.of(POLICY_D), Set.of(), true)
             .getCacheKeyComponent(),
         "Expected differences with ownership policy");
 
     assertNotEquals(
-        ActorContext.asSessionRestricted(userAuth, Set.of(POLICY_D_OWNER_TYPE), Set.of())
+        ActorContext.asSessionRestricted(userAuth, Set.of(POLICY_D_OWNER_TYPE), Set.of(), true)
             .getCacheKeyComponent(),
-        ActorContext.asSessionRestricted(userAuth, Set.of(POLICY_D), Set.of())
+        ActorContext.asSessionRestricted(userAuth, Set.of(POLICY_D), Set.of(), true)
             .getCacheKeyComponent(),
         "Expected differences with ownership type policy");
   }

metadata-operation-context/src/test/java/io/datahubproject/metadata/context/OperationContextTest.java

@@ -27,7 +27,8 @@ public void testSystemPrivilegeEscalation() {
             mock(ServicesRegistryContext.class),
             null,
             TestOperationContexts.emptyActiveUsersRetrieverContext(null),
-            mock(ValidationContext.class));
+            mock(ValidationContext.class),
+            true);
 
     OperationContext opContext =
         systemOpContext.asSession(RequestContext.TEST, Authorizer.EMPTY, userAuth);
@@ -51,7 +52,7 @@ public void testSystemPrivilegeEscalation() {
                 systemOpContext.getOperationContextConfig().toBuilder()
                     .allowSystemAuthentication(false)
                     .build())
-            .build(userAuth);
+            .build(userAuth, true);
 
     assertEquals(
         opContextNoSystem.getAuthentication(),

metadata-service/auth-config/src/main/java/com/datahub/authentication/AuthenticationConfiguration.java

@@ -9,6 +9,9 @@ public class AuthenticationConfiguration {
   /** Whether authentication is enabled */
   private boolean enabled;
 
+  /** Whether user existence is enforced */
+  private boolean enforceExistenceEnabled;
+
   /**
    * List of configurations for {@link com.datahub.plugins.auth.authentication.Authenticator}s to be
    * registered

metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java

@@ -320,7 +320,8 @@ public void setupTest() throws Exception {
             mock(ServicesRegistryContext.class),
             mock(IndexConvention.class),
             mock(RetrieverContext.class),
-            mock(ValidationContext.class));
+            mock(ValidationContext.class),
+            true);
 
     _dataHubAuthorizer =
         new DataHubAuthorizer(

metadata-service/configuration/src/main/resources/application.yaml

@@ -6,6 +6,9 @@ authentication:
   # Enable if you want all requests to the Metadata Service to be authenticated.
   enabled: ${METADATA_SERVICE_AUTH_ENABLED:true}
 
+  # Disable if you want to skip validation of deleted user's tokens
+  enforceExistenceEnabled: ${METADATA_SERVICE_AUTH_ENFORCE_EXISTENCE_ENABLED:true}
+
   # Required if enabled is true! A configurable chain of Authenticators
   authenticators:
     # Required for authenticating requests with DataHub-issued Access Tokens - best not to remove.

metadata-service/factories/src/main/java/com/linkedin/gms/factory/context/SystemOperationContextFactory.java

@@ -79,7 +79,8 @@ protected OperationContext javaSystemOperationContext(
             ValidationContext.builder()
                 .alternateValidation(
                     configurationProvider.getFeatureFlags().isAlternateMCPValidation())
-                .build());
+                .build(),
+            configurationProvider.getAuthentication().isEnforceExistenceEnabled());
 
     entityClientAspectRetriever.setSystemOperationContext(systemOperationContext);
     entityServiceAspectRetriever.setSystemOperationContext(systemOperationContext);
@@ -134,7 +135,8 @@ protected OperationContext restliSystemOperationContext(
             ValidationContext.builder()
                 .alternateValidation(
                     configurationProvider.getFeatureFlags().isAlternateMCPValidation())
-                .build());
+                .build(),
+            configurationProvider.getAuthentication().isEnforceExistenceEnabled());
 
     entityClientAspectRetriever.setSystemOperationContext(systemOperationContext);
     systemGraphRetriever.setSystemOperationContext(systemOperationContext);

metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStepTest.java

@@ -87,7 +87,7 @@ public void testExecuteChecksKeySpecForAllUrns() throws Exception {
     mockOpContext =
         mockOpContext.toBuilder()
             .entityRegistryContext(spyEntityRegistryContext)
-            .build(mockOpContext.getSessionAuthentication());
+            .build(mockOpContext.getSessionAuthentication(), true);
 
     mockDBWithWorkToDo(migrationsDao, countOfCorpUserEntities, countOfChartEntities);