Skip to content

Commit 7df1de0

Browse files
david-leifkerdavid-leifker
committed
wip access controls
TODO: * cache keys - need to be context aware to prevent incorrect results * ownership migration upgrade step * complete unit tests for access controls * restricted entity hydration and graphql response (chris)
1 parent 4a44be8 commit 7df1de0

File tree

37 files changed

+334
-136
lines changed

37 files changed

+334
-136
lines changed

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/health/EntityHealthResolver.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ private Health computeIncidentsHealthForAsset(
138138
final Filter filter = buildIncidentsEntityFilter(entityUrn, IncidentState.ACTIVE.toString());
139139
final SearchResult searchResult =
140140
_entityClient.filter(
141-
Constants.INCIDENT_ENTITY_NAME, filter, null, 0, 1, context.getAuthentication());
141+
context.getOperationContext(), Constants.INCIDENT_ENTITY_NAME, filter, null, 0, 1);
142142
final Integer activeIncidentCount = searchResult.getNumEntities();
143143
if (activeIncidentCount > 0) {
144144
// There are active incidents.

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/incident/EntityIncidentsResolver.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,12 @@ public CompletableFuture<EntityIncidentsResult> get(DataFetchingEnvironment envi
6262
final SortCriterion sortCriterion = buildIncidentsSortCriterion();
6363
final SearchResult searchResult =
6464
_entityClient.filter(
65+
context.getOperationContext(),
6566
Constants.INCIDENT_ENTITY_NAME,
6667
filter,
6768
sortCriterion,
6869
start,
69-
count,
70-
context.getAuthentication());
70+
count);
7171

7272
final List<Urn> incidentUrns =
7373
searchResult.getEntities().stream()

‎datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/incident/EntityIncidentsResolverTest.java

+5-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.linkedin.datahub.graphql.resolvers.incident;
22

33
import static com.linkedin.datahub.graphql.resolvers.incident.EntityIncidentsResolver.*;
4+
import static org.mockito.Mockito.mock;
45
import static org.testng.Assert.*;
56

67
import com.datahub.authentication.Authentication;
@@ -34,6 +35,7 @@
3435
import com.linkedin.metadata.search.SearchResult;
3536
import com.linkedin.metadata.search.utils.QueryUtils;
3637
import graphql.schema.DataFetchingEnvironment;
38+
import io.datahubproject.metadata.context.OperationContext;
3739
import java.util.HashMap;
3840
import java.util.Map;
3941
import org.mockito.Mockito;
@@ -86,12 +88,12 @@ public void testGetSuccess() throws Exception {
8688

8789
Mockito.when(
8890
mockClient.filter(
91+
Mockito.any(OperationContext.class),
8992
Mockito.eq(Constants.INCIDENT_ENTITY_NAME),
9093
Mockito.eq(expectedFilter),
9194
Mockito.eq(expectedSort),
9295
Mockito.eq(0),
93-
Mockito.eq(10),
94-
Mockito.any(Authentication.class)))
96+
Mockito.eq(10)))
9597
.thenReturn(
9698
new SearchResult()
9799
.setFrom(0)
@@ -120,6 +122,7 @@ public void testGetSuccess() throws Exception {
120122
// Execute resolver
121123
QueryContext mockContext = Mockito.mock(QueryContext.class);
122124
Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class));
125+
Mockito.when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class));
123126
DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
124127

125128
Mockito.when(mockEnv.getArgumentOrDefault(Mockito.eq("start"), Mockito.eq(0))).thenReturn(0);

‎entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/ChangeMCP.java

+10-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.linkedin.metadata.aspect.batch;
22

33
import com.linkedin.data.DataMap;
4+
import com.linkedin.data.template.RecordTemplate;
45
import com.linkedin.metadata.aspect.SystemAspect;
56
import java.lang.reflect.InvocationTargetException;
67
import javax.annotation.Nonnull;
@@ -23,6 +24,14 @@ public interface ChangeMCP extends MCPItem {
2324

2425
void setNextAspectVersion(long nextAspectVersion);
2526

27+
@Nullable
28+
default RecordTemplate getPreviousRecordTemplate() {
29+
if (getPreviousSystemAspect() != null) {
30+
return getPreviousSystemAspect().getRecordTemplate();
31+
}
32+
return null;
33+
}
34+
2635
default <T> T getPreviousAspect(Class<T> clazz) {
2736
if (getPreviousSystemAspect() != null) {
2837
try {
@@ -35,8 +44,7 @@ default <T> T getPreviousAspect(Class<T> clazz) {
3544
| NoSuchMethodException e) {
3645
throw new RuntimeException(e);
3746
}
38-
} else {
39-
return null;
4047
}
48+
return null;
4149
}
4250
}

‎entity-registry/src/main/java/com/linkedin/metadata/aspect/hooks/OwnerTypeMap.java

+67-38
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,25 @@
33
import static com.linkedin.metadata.Constants.DEFAULT_OWNERSHIP_TYPE_URN;
44
import static com.linkedin.metadata.Constants.OWNERSHIP_ASPECT_NAME;
55

6-
import com.linkedin.common.AuditStamp;
76
import com.linkedin.common.Owner;
87
import com.linkedin.common.Ownership;
98
import com.linkedin.common.UrnArray;
109
import com.linkedin.common.UrnArrayMap;
1110
import com.linkedin.common.urn.Urn;
1211
import com.linkedin.data.template.RecordTemplate;
13-
import com.linkedin.events.metadata.ChangeType;
12+
import com.linkedin.metadata.aspect.AspectRetriever;
13+
import com.linkedin.metadata.aspect.batch.ChangeMCP;
1414
import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
1515
import com.linkedin.metadata.aspect.plugins.hooks.MutationHook;
16-
import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever;
17-
import com.linkedin.metadata.models.AspectSpec;
18-
import com.linkedin.metadata.models.EntitySpec;
19-
import com.linkedin.mxe.SystemMetadata;
2016
import com.linkedin.util.Pair;
17+
import java.util.Collection;
18+
import java.util.Collections;
19+
import java.util.LinkedList;
20+
import java.util.List;
2121
import java.util.Map;
2222
import java.util.Set;
2323
import java.util.stream.Collectors;
24+
import java.util.stream.Stream;
2425
import javax.annotation.Nonnull;
2526
import javax.annotation.Nullable;
2627

@@ -31,42 +32,70 @@ public OwnerTypeMap(AspectPluginConfig aspectPluginConfig) {
3132
}
3233

3334
@Override
34-
protected void mutate(
35-
@Nonnull ChangeType changeType,
36-
@Nonnull EntitySpec entitySpec,
37-
@Nonnull AspectSpec aspectSpec,
38-
@Nullable RecordTemplate oldAspectValue,
39-
@Nullable RecordTemplate newAspectValue,
40-
@Nullable SystemMetadata oldSystemMetadata,
41-
@Nullable SystemMetadata newSystemMetadata,
42-
@Nonnull AuditStamp auditStamp,
43-
@Nonnull AspectRetriever aspectRetriever) {
44-
if (OWNERSHIP_ASPECT_NAME.equals(aspectSpec.getName()) && newAspectValue != null) {
45-
Ownership ownership = new Ownership(newAspectValue.data());
46-
if (!ownership.getOwners().isEmpty()) {
35+
protected Stream<Pair<ChangeMCP, Boolean>> writeMutation(
36+
@Nonnull Collection<ChangeMCP> changeMCPS, @Nonnull AspectRetriever aspectRetriever) {
37+
38+
List<Pair<ChangeMCP, Boolean>> results = new LinkedList<>();
39+
40+
for (ChangeMCP item : changeMCPS) {
41+
if (OWNERSHIP_ASPECT_NAME.equals(item.getAspectName()) && item.getRecordTemplate() != null) {
42+
final Map<Urn, Set<Owner>> oldOwnerTypes = groupByOwner(item.getPreviousRecordTemplate());
43+
final Map<Urn, Set<Owner>> newOwnerTypes = groupByOwner(item.getRecordTemplate());
44+
45+
Set<Urn> removed =
46+
oldOwnerTypes.keySet().stream()
47+
.filter(owner -> !newOwnerTypes.containsKey(owner))
48+
.collect(Collectors.toSet());
49+
50+
Set<Urn> updated = newOwnerTypes.keySet();
51+
52+
Map<String, UrnArray> ownerTypes =
53+
Stream.concat(removed.stream(), updated.stream())
54+
.map(
55+
ownerUrn -> {
56+
final String ownerFieldName = encodeFieldName(ownerUrn.toString());
57+
if (removed.contains(ownerUrn)) {
58+
// removed
59+
return Pair.of(ownerFieldName, new UrnArray());
60+
}
61+
// updated
62+
return Pair.of(
63+
ownerFieldName,
64+
new UrnArray(
65+
newOwnerTypes.getOrDefault(ownerUrn, Collections.emptySet()).stream()
66+
.map(
67+
owner ->
68+
owner.getTypeUrn() != null
69+
? owner.getTypeUrn()
70+
: DEFAULT_OWNERSHIP_TYPE_URN)
71+
.collect(Collectors.toSet())));
72+
})
73+
.collect(Collectors.toMap(Pair::getFirst, Pair::getSecond));
4774

48-
Map<Urn, Set<Owner>> ownerTypes =
49-
ownership.getOwners().stream()
50-
.collect(Collectors.groupingBy(Owner::getOwner, Collectors.toSet()));
75+
if (!ownerTypes.isEmpty()) {
76+
item.getAspect(Ownership.class).setOwnerTypes(new UrnArrayMap(ownerTypes));
77+
results.add(Pair.of(item, true));
78+
continue;
79+
}
80+
}
81+
82+
// no op
83+
results.add(Pair.of(item, false));
84+
}
5185

52-
ownership.setOwnerTypes(
53-
new UrnArrayMap(
54-
ownerTypes.entrySet().stream()
55-
.map(
56-
entry ->
57-
Pair.of(
58-
encodeFieldName(entry.getKey().toString()),
59-
new UrnArray(
60-
entry.getValue().stream()
61-
.map(
62-
owner ->
63-
owner.getTypeUrn() != null
64-
? owner.getTypeUrn()
65-
: DEFAULT_OWNERSHIP_TYPE_URN)
66-
.collect(Collectors.toSet()))))
67-
.collect(Collectors.toMap(Pair::getKey, Pair::getValue))));
86+
return results.stream();
87+
}
88+
89+
private static Map<Urn, Set<Owner>> groupByOwner(
90+
@Nullable RecordTemplate ownershipRecordTemplate) {
91+
if (ownershipRecordTemplate != null) {
92+
Ownership ownership = new Ownership(ownershipRecordTemplate.data());
93+
if (!ownership.getOwners().isEmpty()) {
94+
return ownership.getOwners().stream()
95+
.collect(Collectors.groupingBy(Owner::getOwner, Collectors.toSet()));
6896
}
6997
}
98+
return Collections.emptyMap();
7099
}
71100

72101
public static String encodeFieldName(String value) {

‎entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ public class SearchableAnnotation {
2020

2121
public static final String FIELD_NAME_ALIASES = "fieldNameAliases";
2222
public static final String ANNOTATION_NAME = "Searchable";
23+
public static final Set<FieldType> OBJECT_FIELD_TYPES =
24+
ImmutableSet.of(FieldType.OBJECT, FieldType.MAP_ARRAY);
2325
private static final Set<FieldType> DEFAULT_QUERY_FIELD_TYPES =
2426
ImmutableSet.of(
2527
FieldType.TEXT,
@@ -71,7 +73,8 @@ public enum FieldType {
7173
OBJECT,
7274
BROWSE_PATH_V2,
7375
WORD_GRAM,
74-
DOUBLE
76+
DOUBLE,
77+
MAP_ARRAY
7578
}
7679

7780
@Nonnull

‎metadata-auth/auth-api/src/main/java/com/datahub/authorization/config/SearchAuthorizationConfiguration.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
package com.datahub.authorization.config;
22

3+
import lombok.AccessLevel;
4+
import lombok.AllArgsConstructor;
35
import lombok.Builder;
46
import lombok.Data;
7+
import lombok.NoArgsConstructor;
58

6-
@Builder
9+
@Builder(toBuilder = true)
710
@Data
11+
@AllArgsConstructor(access = AccessLevel.PACKAGE)
12+
@NoArgsConstructor(access = AccessLevel.PACKAGE)
813
public class SearchAuthorizationConfiguration {
914
private boolean enabled;
1015
}

‎metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ def get_resource_owners_work_unit(
224224

225225
if not current_ownership:
226226
# If we want to overwrite or there are no existing tags, create a new GlobalTags object
227-
current_ownership = OwnershipClass(owners, get_audit_stamp())
227+
current_ownership = OwnershipClass(owners, lastModified=get_audit_stamp())
228228
else:
229229
current_owner_urns: Set[str] = set(
230230
[owner.owner for owner in current_ownership.owners]

‎metadata-ingestion/tests/unit/test_rest_sink.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@
235235
"changeType": "UPSERT",
236236
"aspectName": "ownership",
237237
"aspect": {
238-
"value": '{"owners": [{"owner": "urn:li:corpuser:fbar", "type": "DATAOWNER"}], "lastModified": {"time": 0, "actor": "urn:li:corpuser:fbar"}}',
238+
"value": '{"owners": [{"owner": "urn:li:corpuser:fbar", "type": "DATAOWNER"}], "ownerTypes": {}, "lastModified": {"time": 0, "actor": "urn:li:corpuser:fbar"}}',
239239
"contentType": "application/json",
240240
},
241241
}

‎metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java

+3
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,9 @@ public Pair<Map<String, Set<String>>, List<ChangeMCP>> toUpsertBatchItems(
6363
upsertItem = patchBatchItem.applyPatch(currentValue, aspectRetriever);
6464
}
6565

66+
// Populate old aspect for write hooks
67+
upsertItem.setPreviousSystemAspect(latest);
68+
6669
return upsertItem;
6770
})
6871
.collect(Collectors.toCollection(LinkedList::new));

‎metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java

+34-9
Original file line numberDiff line numberDiff line change
@@ -190,16 +190,18 @@ public SearchResult filter(
190190
@Nonnull OperationContext opContext,
191191
@Nonnull String entityName,
192192
@Nullable Filter filters,
193-
@Nonnull SearchFlags searchFlags,
193+
@Nullable SearchFlags searchFlags,
194194
@Nullable SortCriterion sortCriterion,
195195
int from,
196196
int size) {
197197
log.debug(
198198
String.format(
199199
"Filtering Search documents entityName: %s, filters: %s, sortCriterion: %s, from: %s, size: %s",
200200
entityName, filters, sortCriterion, from, size));
201+
SearchFlags finalSearchFlags =
202+
applyDefaultSearchFlags(searchFlags, null, DEFAULT_SERVICE_SEARCH_FLAGS);
201203
return esSearchDAO.filter(
202-
opContext, entityName, filters, searchFlags, sortCriterion, from, size);
204+
opContext, entityName, filters, finalSearchFlags, sortCriterion, from, size);
203205
}
204206

205207
@Nonnull
@@ -317,10 +319,19 @@ public ScrollResult fullTextScroll(
317319
String.format(
318320
"Scrolling Structured Search documents entities: %s, input: %s, postFilters: %s, sortCriterion: %s, scrollId: %s, size: %s",
319321
entities, input, postFilters, sortCriterion, scrollId, size));
320-
SearchFlags flags = Optional.ofNullable(searchFlags).orElse(new SearchFlags());
321-
flags.setFulltext(true);
322+
SearchFlags finalSearchFlags =
323+
applyDefaultSearchFlags(searchFlags, null, DEFAULT_SERVICE_SEARCH_FLAGS);
324+
finalSearchFlags.setFulltext(true);
322325
return esSearchDAO.scroll(
323-
opContext, entities, input, postFilters, sortCriterion, scrollId, keepAlive, size, flags);
326+
opContext,
327+
entities,
328+
input,
329+
postFilters,
330+
sortCriterion,
331+
scrollId,
332+
keepAlive,
333+
size,
334+
finalSearchFlags);
324335
}
325336

326337
@Nonnull
@@ -339,10 +350,19 @@ public ScrollResult structuredScroll(
339350
String.format(
340351
"Scrolling FullText Search documents entities: %s, input: %s, postFilters: %s, sortCriterion: %s, scrollId: %s, size: %s",
341352
entities, input, postFilters, sortCriterion, scrollId, size));
342-
SearchFlags flags = Optional.ofNullable(searchFlags).orElse(new SearchFlags());
343-
flags.setFulltext(false);
353+
SearchFlags finalSearchFlags =
354+
applyDefaultSearchFlags(searchFlags, null, DEFAULT_SERVICE_SEARCH_FLAGS);
355+
finalSearchFlags.setFulltext(false);
344356
return esSearchDAO.scroll(
345-
opContext, entities, input, postFilters, sortCriterion, scrollId, keepAlive, size, flags);
357+
opContext,
358+
entities,
359+
input,
360+
postFilters,
361+
sortCriterion,
362+
scrollId,
363+
keepAlive,
364+
size,
365+
finalSearchFlags);
346366
}
347367

348368
public Optional<SearchResponse> raw(@Nonnull String indexName, @Nullable String jsonQuery) {
@@ -356,6 +376,7 @@ public int maxResultSize() {
356376

357377
@Override
358378
public ExplainResponse explain(
379+
@Nonnull OperationContext opContext,
359380
@Nonnull String query,
360381
@Nonnull String documentId,
361382
@Nonnull String entityName,
@@ -366,13 +387,17 @@ public ExplainResponse explain(
366387
@Nullable String keepAlive,
367388
int size,
368389
@Nullable List<String> facets) {
390+
SearchFlags finalSearchFlags =
391+
applyDefaultSearchFlags(searchFlags, null, DEFAULT_SERVICE_SEARCH_FLAGS);
392+
369393
return esSearchDAO.explain(
394+
opContext,
370395
query,
371396
documentId,
372397
entityName,
373398
postFilters,
374399
sortCriterion,
375-
searchFlags,
400+
finalSearchFlags,
376401
scrollId,
377402
keepAlive,
378403
size,

0 commit comments

Comments
 (0)