Skip to content

Commit 718813a

Browse files
david-leifkerdavid-leifker
committed
feat(search): search access controls part 1
TODO: * ownership migration upgrade step * complete unit tests for access controls * comprehensive api coveration graphql, openapi, restli * restricted entity hydration and graphql response
1 parent 4a44be8 commit 718813a

File tree

67 files changed

+688
-303
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+688
-303
lines changed

‎build.gradle

+3-3
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ buildscript {
1717

1818
ext.javaClassVersion = { p ->
1919
// If Spring 6 is present, hard dependency on jdk17
20-
if (p.configurations.any { it.getDependencies().any{
20+
if (p.configurations.any { it.getDependencies().any {
2121
(it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6."))
2222
|| (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test"))
2323
}}) {
@@ -43,7 +43,7 @@ buildscript {
4343
ext.elasticsearchVersion = '2.9.0' // ES 7.10, Opensearch 1.x, 2.x
4444
ext.jacksonVersion = '2.15.3'
4545
ext.jettyVersion = '11.0.19'
46-
ext.playVersion = '2.8.18'
46+
ext.playVersion = '2.8.21'
4747
ext.log4jVersion = '2.19.0'
4848
ext.slf4jVersion = '1.7.36'
4949
ext.logbackClassic = '1.4.14'
@@ -132,7 +132,7 @@ project.ext.externalDependency = [
132132
'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:21.0',
133133
'gson': 'com.google.code.gson:gson:2.8.9',
134134
'guice': 'com.google.inject:guice:7.0.0',
135-
'guice4': 'com.google.inject:guice:4.2.3', // Used for frontend while still on old Play version
135+
'guicePlay': 'com.google.inject:guice:5.0.1', // Used for frontend while still on old Play version
136136
'guava': 'com.google.guava:guava:32.1.2-jre',
137137
'h2': 'com.h2database:h2:2.2.224',
138138
'hadoopCommon':'org.apache.hadoop:hadoop-common:2.7.2',

‎datahub-frontend/app/auth/AuthModule.java

+41-2
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,28 @@
88
import com.datahub.authentication.Actor;
99
import com.datahub.authentication.ActorType;
1010
import com.datahub.authentication.Authentication;
11+
import com.datahub.plugins.auth.authorization.Authorizer;
1112
import com.google.inject.AbstractModule;
1213
import com.google.inject.Provides;
1314
import com.google.inject.Singleton;
15+
import com.google.inject.name.Named;
1416
import com.linkedin.entity.client.SystemEntityClient;
1517
import com.linkedin.entity.client.SystemRestliEntityClient;
1618
import com.linkedin.metadata.restli.DefaultRestliClientFactory;
19+
import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl;
1720
import com.linkedin.parseq.retry.backoff.ExponentialBackoff;
1821
import com.linkedin.util.Configuration;
1922
import config.ConfigurationProvider;
2023
import controllers.SsoCallbackController;
2124
import java.nio.charset.StandardCharsets;
2225
import java.util.Collections;
26+
27+
import io.datahubproject.metadata.context.ActorContext;
28+
import io.datahubproject.metadata.context.AuthorizerContext;
29+
import io.datahubproject.metadata.context.EntityRegistryContext;
30+
import io.datahubproject.metadata.context.OperationContext;
31+
import io.datahubproject.metadata.context.OperationContextConfig;
32+
import io.datahubproject.metadata.context.SearchContext;
2333
import lombok.extern.slf4j.Slf4j;
2434
import org.apache.commons.codec.digest.DigestUtils;
2535
import org.apache.http.impl.client.CloseableHttpClient;
@@ -32,11 +42,15 @@
3242
import org.pac4j.play.store.PlayCookieSessionStore;
3343
import org.pac4j.play.store.PlaySessionStore;
3444
import org.pac4j.play.store.ShiroAesDataEncrypter;
45+
import org.springframework.beans.factory.annotation.Qualifier;
3546
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
47+
import org.springframework.context.annotation.Bean;
3648
import play.Environment;
3749
import play.cache.SyncCacheApi;
3850
import utils.ConfigUtil;
3951

52+
import javax.annotation.Nonnull;
53+
4054
/** Responsible for configuring, validating, and providing authentication related components. */
4155
@Slf4j
4256
public class AuthModule extends AbstractModule {
@@ -152,6 +166,31 @@ protected Authentication provideSystemAuthentication() {
152166
Collections.emptyMap());
153167
}
154168

169+
@Provides
170+
@Singleton
171+
@Named("systemOperationContext")
172+
protected OperationContext provideOperationContext(final Authentication systemAuthentication,
173+
final ConfigurationProvider configurationProvider) {
174+
ActorContext systemActorContext =
175+
ActorContext.builder()
176+
.systemAuthentication(true)
177+
.authentication(systemAuthentication)
178+
.build();
179+
OperationContextConfig systemConfig = OperationContextConfig.builder()
180+
.searchAuthorizationConfiguration(configurationProvider.getAuthorization().getSearch())
181+
.allowSystemAuthentication(true)
182+
.build();
183+
184+
return OperationContext.builder()
185+
.operationContextConfig(systemConfig)
186+
.systemActorContext(systemActorContext)
187+
.searchContext(SearchContext.EMPTY)
188+
.entityRegistryContext(EntityRegistryContext.EMPTY)
189+
// Authorizer.EMPTY doesn't actually apply to system auth
190+
.authorizerContext(AuthorizerContext.builder().authorizer(Authorizer.EMPTY).build())
191+
.build(systemAuthentication);
192+
}
193+
155194
@Provides
156195
@Singleton
157196
protected ConfigurationProvider provideConfigurationProvider() {
@@ -163,13 +202,13 @@ protected ConfigurationProvider provideConfigurationProvider() {
163202
@Provides
164203
@Singleton
165204
protected SystemEntityClient provideEntityClient(
166-
final Authentication systemAuthentication,
205+
@Named("systemOperationContext") final OperationContext systemOperationContext,
167206
final ConfigurationProvider configurationProvider) {
168207
return new SystemRestliEntityClient(
208+
systemOperationContext,
169209
buildRestliClient(),
170210
new ExponentialBackoff(_configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)),
171211
_configs.getInt(ENTITY_CLIENT_NUM_RETRIES),
172-
systemAuthentication,
173212
configurationProvider.getCache().getClient().getEntityClient());
174213
}
175214

‎datahub-frontend/app/config/ConfigurationProvider.java

+4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package config;
22

3+
import com.datahub.authorization.AuthorizationConfiguration;
34
import com.linkedin.metadata.config.VisualConfiguration;
45
import com.linkedin.metadata.config.cache.CacheConfiguration;
56
import com.linkedin.metadata.config.kafka.KafkaConfiguration;
@@ -26,4 +27,7 @@ public class ConfigurationProvider {
2627

2728
/** Configuration for the view layer */
2829
private VisualConfiguration visualConfig;
30+
31+
/** Configuration for authorization */
32+
private AuthorizationConfiguration authorization;
2933
}

‎datahub-frontend/play.gradle

+2-2
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ dependencies {
7676

7777
implementation externalDependency.slf4jApi
7878
compileOnly externalDependency.lombok
79-
runtimeOnly externalDependency.guice4
79+
runtimeOnly externalDependency.guicePlay
8080
runtimeOnly (externalDependency.playDocs) {
8181
exclude group: 'com.typesafe.akka', module: 'akka-http-core_2.12'
8282
}
@@ -90,7 +90,7 @@ dependencies {
9090

9191
play {
9292
platform {
93-
playVersion = '2.8.18'
93+
playVersion = '2.8.21'
9494
scalaVersion = '2.12'
9595
javaVersion = JavaVersion.VERSION_11
9696
}

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/health/EntityHealthResolver.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ private Health computeIncidentsHealthForAsset(
138138
final Filter filter = buildIncidentsEntityFilter(entityUrn, IncidentState.ACTIVE.toString());
139139
final SearchResult searchResult =
140140
_entityClient.filter(
141-
Constants.INCIDENT_ENTITY_NAME, filter, null, 0, 1, context.getAuthentication());
141+
context.getOperationContext(), Constants.INCIDENT_ENTITY_NAME, filter, null, 0, 1);
142142
final Integer activeIncidentCount = searchResult.getNumEntities();
143143
if (activeIncidentCount > 0) {
144144
// There are active incidents.

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/incident/EntityIncidentsResolver.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,12 @@ public CompletableFuture<EntityIncidentsResult> get(DataFetchingEnvironment envi
6262
final SortCriterion sortCriterion = buildIncidentsSortCriterion();
6363
final SearchResult searchResult =
6464
_entityClient.filter(
65+
context.getOperationContext(),
6566
Constants.INCIDENT_ENTITY_NAME,
6667
filter,
6768
sortCriterion,
6869
start,
69-
count,
70-
context.getAuthentication());
70+
count);
7171

7272
final List<Urn> incidentUrns =
7373
searchResult.getEntities().stream()

‎datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/incident/EntityIncidentsResolverTest.java

+5-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.linkedin.datahub.graphql.resolvers.incident;
22

33
import static com.linkedin.datahub.graphql.resolvers.incident.EntityIncidentsResolver.*;
4+
import static org.mockito.Mockito.mock;
45
import static org.testng.Assert.*;
56

67
import com.datahub.authentication.Authentication;
@@ -34,6 +35,7 @@
3435
import com.linkedin.metadata.search.SearchResult;
3536
import com.linkedin.metadata.search.utils.QueryUtils;
3637
import graphql.schema.DataFetchingEnvironment;
38+
import io.datahubproject.metadata.context.OperationContext;
3739
import java.util.HashMap;
3840
import java.util.Map;
3941
import org.mockito.Mockito;
@@ -86,12 +88,12 @@ public void testGetSuccess() throws Exception {
8688

8789
Mockito.when(
8890
mockClient.filter(
91+
Mockito.any(OperationContext.class),
8992
Mockito.eq(Constants.INCIDENT_ENTITY_NAME),
9093
Mockito.eq(expectedFilter),
9194
Mockito.eq(expectedSort),
9295
Mockito.eq(0),
93-
Mockito.eq(10),
94-
Mockito.any(Authentication.class)))
96+
Mockito.eq(10)))
9597
.thenReturn(
9698
new SearchResult()
9799
.setFrom(0)
@@ -120,6 +122,7 @@ public void testGetSuccess() throws Exception {
120122
// Execute resolver
121123
QueryContext mockContext = Mockito.mock(QueryContext.class);
122124
Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class));
125+
Mockito.when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class));
123126
DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
124127

125128
Mockito.when(mockEnv.getArgumentOrDefault(Mockito.eq("start"), Mockito.eq(0))).thenReturn(0);

‎entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/ChangeMCP.java

+10-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.linkedin.metadata.aspect.batch;
22

33
import com.linkedin.data.DataMap;
4+
import com.linkedin.data.template.RecordTemplate;
45
import com.linkedin.metadata.aspect.SystemAspect;
56
import java.lang.reflect.InvocationTargetException;
67
import javax.annotation.Nonnull;
@@ -23,6 +24,14 @@ public interface ChangeMCP extends MCPItem {
2324

2425
void setNextAspectVersion(long nextAspectVersion);
2526

27+
@Nullable
28+
default RecordTemplate getPreviousRecordTemplate() {
29+
if (getPreviousSystemAspect() != null) {
30+
return getPreviousSystemAspect().getRecordTemplate();
31+
}
32+
return null;
33+
}
34+
2635
default <T> T getPreviousAspect(Class<T> clazz) {
2736
if (getPreviousSystemAspect() != null) {
2837
try {
@@ -35,8 +44,7 @@ default <T> T getPreviousAspect(Class<T> clazz) {
3544
| NoSuchMethodException e) {
3645
throw new RuntimeException(e);
3746
}
38-
} else {
39-
return null;
4047
}
48+
return null;
4149
}
4250
}

‎entity-registry/src/main/java/com/linkedin/metadata/aspect/hooks/OwnerTypeMap.java

+67-38
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,25 @@
33
import static com.linkedin.metadata.Constants.DEFAULT_OWNERSHIP_TYPE_URN;
44
import static com.linkedin.metadata.Constants.OWNERSHIP_ASPECT_NAME;
55

6-
import com.linkedin.common.AuditStamp;
76
import com.linkedin.common.Owner;
87
import com.linkedin.common.Ownership;
98
import com.linkedin.common.UrnArray;
109
import com.linkedin.common.UrnArrayMap;
1110
import com.linkedin.common.urn.Urn;
1211
import com.linkedin.data.template.RecordTemplate;
13-
import com.linkedin.events.metadata.ChangeType;
12+
import com.linkedin.metadata.aspect.AspectRetriever;
13+
import com.linkedin.metadata.aspect.batch.ChangeMCP;
1414
import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
1515
import com.linkedin.metadata.aspect.plugins.hooks.MutationHook;
16-
import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever;
17-
import com.linkedin.metadata.models.AspectSpec;
18-
import com.linkedin.metadata.models.EntitySpec;
19-
import com.linkedin.mxe.SystemMetadata;
2016
import com.linkedin.util.Pair;
17+
import java.util.Collection;
18+
import java.util.Collections;
19+
import java.util.LinkedList;
20+
import java.util.List;
2121
import java.util.Map;
2222
import java.util.Set;
2323
import java.util.stream.Collectors;
24+
import java.util.stream.Stream;
2425
import javax.annotation.Nonnull;
2526
import javax.annotation.Nullable;
2627

@@ -31,42 +32,70 @@ public OwnerTypeMap(AspectPluginConfig aspectPluginConfig) {
3132
}
3233

3334
@Override
34-
protected void mutate(
35-
@Nonnull ChangeType changeType,
36-
@Nonnull EntitySpec entitySpec,
37-
@Nonnull AspectSpec aspectSpec,
38-
@Nullable RecordTemplate oldAspectValue,
39-
@Nullable RecordTemplate newAspectValue,
40-
@Nullable SystemMetadata oldSystemMetadata,
41-
@Nullable SystemMetadata newSystemMetadata,
42-
@Nonnull AuditStamp auditStamp,
43-
@Nonnull AspectRetriever aspectRetriever) {
44-
if (OWNERSHIP_ASPECT_NAME.equals(aspectSpec.getName()) && newAspectValue != null) {
45-
Ownership ownership = new Ownership(newAspectValue.data());
46-
if (!ownership.getOwners().isEmpty()) {
35+
protected Stream<Pair<ChangeMCP, Boolean>> writeMutation(
36+
@Nonnull Collection<ChangeMCP> changeMCPS, @Nonnull AspectRetriever aspectRetriever) {
37+
38+
List<Pair<ChangeMCP, Boolean>> results = new LinkedList<>();
39+
40+
for (ChangeMCP item : changeMCPS) {
41+
if (OWNERSHIP_ASPECT_NAME.equals(item.getAspectName()) && item.getRecordTemplate() != null) {
42+
final Map<Urn, Set<Owner>> oldOwnerTypes = groupByOwner(item.getPreviousRecordTemplate());
43+
final Map<Urn, Set<Owner>> newOwnerTypes = groupByOwner(item.getRecordTemplate());
44+
45+
Set<Urn> removed =
46+
oldOwnerTypes.keySet().stream()
47+
.filter(owner -> !newOwnerTypes.containsKey(owner))
48+
.collect(Collectors.toSet());
49+
50+
Set<Urn> updated = newOwnerTypes.keySet();
51+
52+
Map<String, UrnArray> ownerTypes =
53+
Stream.concat(removed.stream(), updated.stream())
54+
.map(
55+
ownerUrn -> {
56+
final String ownerFieldName = encodeFieldName(ownerUrn.toString());
57+
if (removed.contains(ownerUrn)) {
58+
// removed
59+
return Pair.of(ownerFieldName, new UrnArray());
60+
}
61+
// updated
62+
return Pair.of(
63+
ownerFieldName,
64+
new UrnArray(
65+
newOwnerTypes.getOrDefault(ownerUrn, Collections.emptySet()).stream()
66+
.map(
67+
owner ->
68+
owner.getTypeUrn() != null
69+
? owner.getTypeUrn()
70+
: DEFAULT_OWNERSHIP_TYPE_URN)
71+
.collect(Collectors.toSet())));
72+
})
73+
.collect(Collectors.toMap(Pair::getFirst, Pair::getSecond));
4774

48-
Map<Urn, Set<Owner>> ownerTypes =
49-
ownership.getOwners().stream()
50-
.collect(Collectors.groupingBy(Owner::getOwner, Collectors.toSet()));
75+
if (!ownerTypes.isEmpty()) {
76+
item.getAspect(Ownership.class).setOwnerTypes(new UrnArrayMap(ownerTypes));
77+
results.add(Pair.of(item, true));
78+
continue;
79+
}
80+
}
81+
82+
// no op
83+
results.add(Pair.of(item, false));
84+
}
5185

52-
ownership.setOwnerTypes(
53-
new UrnArrayMap(
54-
ownerTypes.entrySet().stream()
55-
.map(
56-
entry ->
57-
Pair.of(
58-
encodeFieldName(entry.getKey().toString()),
59-
new UrnArray(
60-
entry.getValue().stream()
61-
.map(
62-
owner ->
63-
owner.getTypeUrn() != null
64-
? owner.getTypeUrn()
65-
: DEFAULT_OWNERSHIP_TYPE_URN)
66-
.collect(Collectors.toSet()))))
67-
.collect(Collectors.toMap(Pair::getKey, Pair::getValue))));
86+
return results.stream();
87+
}
88+
89+
private static Map<Urn, Set<Owner>> groupByOwner(
90+
@Nullable RecordTemplate ownershipRecordTemplate) {
91+
if (ownershipRecordTemplate != null) {
92+
Ownership ownership = new Ownership(ownershipRecordTemplate.data());
93+
if (!ownership.getOwners().isEmpty()) {
94+
return ownership.getOwners().stream()
95+
.collect(Collectors.groupingBy(Owner::getOwner, Collectors.toSet()));
6896
}
6997
}
98+
return Collections.emptyMap();
7099
}
71100

72101
public static String encodeFieldName(String value) {

0 commit comments

Comments
 (0)