Skip to content

Commit 0522e7e

Browse files
david-leifkerdavid-leifker
committed
wip access controls
TODO: * cache keys - need to be context aware to prevent incorrect results * ownership migration upgrade step * complete unit tests for access controls * restricted entity hydration and graphql response (chris)
1 parent 4a44be8 commit 0522e7e

File tree

29 files changed

+313
-127
lines changed

29 files changed

+313
-127
lines changed

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/health/EntityHealthResolver.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ private Health computeIncidentsHealthForAsset(
138138
final Filter filter = buildIncidentsEntityFilter(entityUrn, IncidentState.ACTIVE.toString());
139139
final SearchResult searchResult =
140140
_entityClient.filter(
141-
Constants.INCIDENT_ENTITY_NAME, filter, null, 0, 1, context.getAuthentication());
141+
context.getOperationContext(), Constants.INCIDENT_ENTITY_NAME, filter, null, 0, 1);
142142
final Integer activeIncidentCount = searchResult.getNumEntities();
143143
if (activeIncidentCount > 0) {
144144
// There are active incidents.

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/incident/EntityIncidentsResolver.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,12 @@ public CompletableFuture<EntityIncidentsResult> get(DataFetchingEnvironment envi
6262
final SortCriterion sortCriterion = buildIncidentsSortCriterion();
6363
final SearchResult searchResult =
6464
_entityClient.filter(
65+
context.getOperationContext(),
6566
Constants.INCIDENT_ENTITY_NAME,
6667
filter,
6768
sortCriterion,
6869
start,
69-
count,
70-
context.getAuthentication());
70+
count);
7171

7272
final List<Urn> incidentUrns =
7373
searchResult.getEntities().stream()

‎datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/incident/EntityIncidentsResolverTest.java

+5-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.linkedin.datahub.graphql.resolvers.incident;
22

33
import static com.linkedin.datahub.graphql.resolvers.incident.EntityIncidentsResolver.*;
4+
import static org.mockito.Mockito.mock;
45
import static org.testng.Assert.*;
56

67
import com.datahub.authentication.Authentication;
@@ -34,6 +35,7 @@
3435
import com.linkedin.metadata.search.SearchResult;
3536
import com.linkedin.metadata.search.utils.QueryUtils;
3637
import graphql.schema.DataFetchingEnvironment;
38+
import io.datahubproject.metadata.context.OperationContext;
3739
import java.util.HashMap;
3840
import java.util.Map;
3941
import org.mockito.Mockito;
@@ -86,12 +88,12 @@ public void testGetSuccess() throws Exception {
8688

8789
Mockito.when(
8890
mockClient.filter(
91+
Mockito.any(OperationContext.class),
8992
Mockito.eq(Constants.INCIDENT_ENTITY_NAME),
9093
Mockito.eq(expectedFilter),
9194
Mockito.eq(expectedSort),
9295
Mockito.eq(0),
93-
Mockito.eq(10),
94-
Mockito.any(Authentication.class)))
96+
Mockito.eq(10)))
9597
.thenReturn(
9698
new SearchResult()
9799
.setFrom(0)
@@ -120,6 +122,7 @@ public void testGetSuccess() throws Exception {
120122
// Execute resolver
121123
QueryContext mockContext = Mockito.mock(QueryContext.class);
122124
Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class));
125+
Mockito.when(mockContext.getOperationContext()).thenReturn(mock(OperationContext.class));
123126
DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
124127

125128
Mockito.when(mockEnv.getArgumentOrDefault(Mockito.eq("start"), Mockito.eq(0))).thenReturn(0);

‎entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/ChangeMCP.java

+10-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.linkedin.metadata.aspect.batch;
22

33
import com.linkedin.data.DataMap;
4+
import com.linkedin.data.template.RecordTemplate;
45
import com.linkedin.metadata.aspect.SystemAspect;
56
import java.lang.reflect.InvocationTargetException;
67
import javax.annotation.Nonnull;
@@ -23,6 +24,14 @@ public interface ChangeMCP extends MCPItem {
2324

2425
void setNextAspectVersion(long nextAspectVersion);
2526

27+
@Nullable
28+
default RecordTemplate getPreviousRecordTemplate() {
29+
if (getPreviousSystemAspect() != null) {
30+
return getPreviousSystemAspect().getRecordTemplate();
31+
}
32+
return null;
33+
}
34+
2635
default <T> T getPreviousAspect(Class<T> clazz) {
2736
if (getPreviousSystemAspect() != null) {
2837
try {
@@ -35,8 +44,7 @@ default <T> T getPreviousAspect(Class<T> clazz) {
3544
| NoSuchMethodException e) {
3645
throw new RuntimeException(e);
3746
}
38-
} else {
39-
return null;
4047
}
48+
return null;
4149
}
4250
}

‎entity-registry/src/main/java/com/linkedin/metadata/aspect/hooks/OwnerTypeMap.java

+63-38
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,24 @@
33
import static com.linkedin.metadata.Constants.DEFAULT_OWNERSHIP_TYPE_URN;
44
import static com.linkedin.metadata.Constants.OWNERSHIP_ASPECT_NAME;
55

6-
import com.linkedin.common.AuditStamp;
76
import com.linkedin.common.Owner;
87
import com.linkedin.common.Ownership;
98
import com.linkedin.common.UrnArray;
109
import com.linkedin.common.UrnArrayMap;
1110
import com.linkedin.common.urn.Urn;
1211
import com.linkedin.data.template.RecordTemplate;
13-
import com.linkedin.events.metadata.ChangeType;
12+
import com.linkedin.metadata.aspect.AspectRetriever;
13+
import com.linkedin.metadata.aspect.batch.ChangeMCP;
1414
import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig;
1515
import com.linkedin.metadata.aspect.plugins.hooks.MutationHook;
16-
import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever;
17-
import com.linkedin.metadata.models.AspectSpec;
18-
import com.linkedin.metadata.models.EntitySpec;
19-
import com.linkedin.mxe.SystemMetadata;
2016
import com.linkedin.util.Pair;
17+
import java.util.Collection;
18+
import java.util.Collections;
19+
import java.util.HashSet;
2120
import java.util.Map;
2221
import java.util.Set;
2322
import java.util.stream.Collectors;
23+
import java.util.stream.Stream;
2424
import javax.annotation.Nonnull;
2525
import javax.annotation.Nullable;
2626

@@ -31,42 +31,67 @@ public OwnerTypeMap(AspectPluginConfig aspectPluginConfig) {
3131
}
3232

3333
@Override
34-
protected void mutate(
35-
@Nonnull ChangeType changeType,
36-
@Nonnull EntitySpec entitySpec,
37-
@Nonnull AspectSpec aspectSpec,
38-
@Nullable RecordTemplate oldAspectValue,
39-
@Nullable RecordTemplate newAspectValue,
40-
@Nullable SystemMetadata oldSystemMetadata,
41-
@Nullable SystemMetadata newSystemMetadata,
42-
@Nonnull AuditStamp auditStamp,
43-
@Nonnull AspectRetriever aspectRetriever) {
44-
if (OWNERSHIP_ASPECT_NAME.equals(aspectSpec.getName()) && newAspectValue != null) {
45-
Ownership ownership = new Ownership(newAspectValue.data());
46-
if (!ownership.getOwners().isEmpty()) {
34+
protected Stream<Pair<ChangeMCP, Boolean>> writeMutation(
35+
@Nonnull Collection<ChangeMCP> changeMCPS, @Nonnull AspectRetriever aspectRetriever) {
36+
return changeMCPS.stream()
37+
.map(
38+
item -> {
39+
if (OWNERSHIP_ASPECT_NAME.equals(item.getAspectName())
40+
&& item.getRecordTemplate() != null) {
41+
final Map<Urn, Set<Owner>> oldOwnerTypes =
42+
groupByOwner(item.getPreviousRecordTemplate());
43+
final Map<Urn, Set<Owner>> newOwnerTypes = groupByOwner(item.getRecordTemplate());
44+
45+
if (!oldOwnerTypes.isEmpty() || !newOwnerTypes.isEmpty()) {
46+
Set<Urn> changedOwners = new HashSet<>(newOwnerTypes.keySet());
47+
changedOwners.addAll(oldOwnerTypes.keySet());
4748

48-
Map<Urn, Set<Owner>> ownerTypes =
49-
ownership.getOwners().stream()
50-
.collect(Collectors.groupingBy(Owner::getOwner, Collectors.toSet()));
49+
item.getAspect(Ownership.class)
50+
.setOwnerTypes(
51+
new UrnArrayMap(
52+
changedOwners.stream()
53+
.map(
54+
ownerUrn -> {
55+
if (!newOwnerTypes.containsKey(ownerUrn)
56+
&& oldOwnerTypes.containsKey(ownerUrn)) {
57+
// removed
58+
return Pair.of(
59+
encodeFieldName(ownerUrn.toString()), new UrnArray());
60+
} else {
61+
return Pair.of(
62+
encodeFieldName(ownerUrn.toString()),
63+
new UrnArray(
64+
newOwnerTypes
65+
.getOrDefault(
66+
ownerUrn, Collections.emptySet())
67+
.stream()
68+
.map(
69+
owner ->
70+
owner.getTypeUrn() != null
71+
? owner.getTypeUrn()
72+
: DEFAULT_OWNERSHIP_TYPE_URN)
73+
.collect(Collectors.toSet())));
74+
}
75+
})
76+
.collect(Collectors.toMap(Pair::getKey, Pair::getValue))));
5177

52-
ownership.setOwnerTypes(
53-
new UrnArrayMap(
54-
ownerTypes.entrySet().stream()
55-
.map(
56-
entry ->
57-
Pair.of(
58-
encodeFieldName(entry.getKey().toString()),
59-
new UrnArray(
60-
entry.getValue().stream()
61-
.map(
62-
owner ->
63-
owner.getTypeUrn() != null
64-
? owner.getTypeUrn()
65-
: DEFAULT_OWNERSHIP_TYPE_URN)
66-
.collect(Collectors.toSet()))))
67-
.collect(Collectors.toMap(Pair::getKey, Pair::getValue))));
78+
return Pair.of(item, true);
79+
}
80+
}
81+
return Pair.of(item, false);
82+
});
83+
}
84+
85+
private static Map<Urn, Set<Owner>> groupByOwner(
86+
@Nullable RecordTemplate ownershipRecordTemplate) {
87+
if (ownershipRecordTemplate != null) {
88+
Ownership ownership = new Ownership(ownershipRecordTemplate.data());
89+
if (!ownership.getOwners().isEmpty()) {
90+
return ownership.getOwners().stream()
91+
.collect(Collectors.groupingBy(Owner::getOwner, Collectors.toSet()));
6892
}
6993
}
94+
return Collections.emptyMap();
7095
}
7196

7297
public static String encodeFieldName(String value) {

‎entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ public class SearchableAnnotation {
2020

2121
public static final String FIELD_NAME_ALIASES = "fieldNameAliases";
2222
public static final String ANNOTATION_NAME = "Searchable";
23+
public static final Set<FieldType> OBJECT_FIELD_TYPES =
24+
ImmutableSet.of(FieldType.OBJECT, FieldType.MAP_ARRAY);
2325
private static final Set<FieldType> DEFAULT_QUERY_FIELD_TYPES =
2426
ImmutableSet.of(
2527
FieldType.TEXT,
@@ -71,7 +73,8 @@ public enum FieldType {
7173
OBJECT,
7274
BROWSE_PATH_V2,
7375
WORD_GRAM,
74-
DOUBLE
76+
DOUBLE,
77+
MAP_ARRAY
7578
}
7679

7780
@Nonnull

‎metadata-auth/auth-api/src/main/java/com/datahub/authorization/config/SearchAuthorizationConfiguration.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
package com.datahub.authorization.config;
22

3+
import lombok.AccessLevel;
4+
import lombok.AllArgsConstructor;
35
import lombok.Builder;
46
import lombok.Data;
7+
import lombok.NoArgsConstructor;
58

6-
@Builder
9+
@Builder(toBuilder = true)
710
@Data
11+
@AllArgsConstructor(access = AccessLevel.PACKAGE)
12+
@NoArgsConstructor(access = AccessLevel.PACKAGE)
813
public class SearchAuthorizationConfiguration {
914
private boolean enabled;
1015
}

‎metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ def get_resource_owners_work_unit(
224224

225225
if not current_ownership:
226226
# If we want to overwrite or there are no existing tags, create a new GlobalTags object
227-
current_ownership = OwnershipClass(owners, get_audit_stamp())
227+
current_ownership = OwnershipClass(owners, lastModified=get_audit_stamp())
228228
else:
229229
current_owner_urns: Set[str] = set(
230230
[owner.owner for owner in current_ownership.owners]

‎metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java

+34-9
Original file line numberDiff line numberDiff line change
@@ -190,16 +190,18 @@ public SearchResult filter(
190190
@Nonnull OperationContext opContext,
191191
@Nonnull String entityName,
192192
@Nullable Filter filters,
193-
@Nonnull SearchFlags searchFlags,
193+
@Nullable SearchFlags searchFlags,
194194
@Nullable SortCriterion sortCriterion,
195195
int from,
196196
int size) {
197197
log.debug(
198198
String.format(
199199
"Filtering Search documents entityName: %s, filters: %s, sortCriterion: %s, from: %s, size: %s",
200200
entityName, filters, sortCriterion, from, size));
201+
SearchFlags finalSearchFlags =
202+
applyDefaultSearchFlags(searchFlags, null, DEFAULT_SERVICE_SEARCH_FLAGS);
201203
return esSearchDAO.filter(
202-
opContext, entityName, filters, searchFlags, sortCriterion, from, size);
204+
opContext, entityName, filters, finalSearchFlags, sortCriterion, from, size);
203205
}
204206

205207
@Nonnull
@@ -317,10 +319,19 @@ public ScrollResult fullTextScroll(
317319
String.format(
318320
"Scrolling Structured Search documents entities: %s, input: %s, postFilters: %s, sortCriterion: %s, scrollId: %s, size: %s",
319321
entities, input, postFilters, sortCriterion, scrollId, size));
320-
SearchFlags flags = Optional.ofNullable(searchFlags).orElse(new SearchFlags());
321-
flags.setFulltext(true);
322+
SearchFlags finalSearchFlags =
323+
applyDefaultSearchFlags(searchFlags, null, DEFAULT_SERVICE_SEARCH_FLAGS);
324+
finalSearchFlags.setFulltext(true);
322325
return esSearchDAO.scroll(
323-
opContext, entities, input, postFilters, sortCriterion, scrollId, keepAlive, size, flags);
326+
opContext,
327+
entities,
328+
input,
329+
postFilters,
330+
sortCriterion,
331+
scrollId,
332+
keepAlive,
333+
size,
334+
finalSearchFlags);
324335
}
325336

326337
@Nonnull
@@ -339,10 +350,19 @@ public ScrollResult structuredScroll(
339350
String.format(
340351
"Scrolling FullText Search documents entities: %s, input: %s, postFilters: %s, sortCriterion: %s, scrollId: %s, size: %s",
341352
entities, input, postFilters, sortCriterion, scrollId, size));
342-
SearchFlags flags = Optional.ofNullable(searchFlags).orElse(new SearchFlags());
343-
flags.setFulltext(false);
353+
SearchFlags finalSearchFlags =
354+
applyDefaultSearchFlags(searchFlags, null, DEFAULT_SERVICE_SEARCH_FLAGS);
355+
finalSearchFlags.setFulltext(false);
344356
return esSearchDAO.scroll(
345-
opContext, entities, input, postFilters, sortCriterion, scrollId, keepAlive, size, flags);
357+
opContext,
358+
entities,
359+
input,
360+
postFilters,
361+
sortCriterion,
362+
scrollId,
363+
keepAlive,
364+
size,
365+
finalSearchFlags);
346366
}
347367

348368
public Optional<SearchResponse> raw(@Nonnull String indexName, @Nullable String jsonQuery) {
@@ -356,6 +376,7 @@ public int maxResultSize() {
356376

357377
@Override
358378
public ExplainResponse explain(
379+
@Nonnull OperationContext opContext,
359380
@Nonnull String query,
360381
@Nonnull String documentId,
361382
@Nonnull String entityName,
@@ -366,13 +387,17 @@ public ExplainResponse explain(
366387
@Nullable String keepAlive,
367388
int size,
368389
@Nullable List<String> facets) {
390+
SearchFlags finalSearchFlags =
391+
applyDefaultSearchFlags(searchFlags, null, DEFAULT_SERVICE_SEARCH_FLAGS);
392+
369393
return esSearchDAO.explain(
394+
opContext,
370395
query,
371396
documentId,
372397
entityName,
373398
postFilters,
374399
sortCriterion,
375-
searchFlags,
400+
finalSearchFlags,
376401
scrollId,
377402
keepAlive,
378403
size,

‎metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import static com.linkedin.metadata.Constants.ENTITY_TYPE_URN_PREFIX;
44
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD;
55
import static com.linkedin.metadata.models.StructuredPropertyUtils.sanitizeStructuredPropertyFQN;
6+
import static com.linkedin.metadata.models.annotation.SearchableAnnotation.OBJECT_FIELD_TYPES;
67
import static com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder.*;
78

89
import com.google.common.collect.ImmutableMap;
@@ -53,6 +54,7 @@ public static Map<String, String> getPartialNgramConfigWithOverrides(
5354
public static final String PATH = "path";
5455

5556
public static final String PROPERTIES = "properties";
57+
public static final String DYNAMIC_TEMPLATES = "dynamic_templates";
5658

5759
private MappingsBuilder() {}
5860

@@ -100,6 +102,7 @@ public static Map<String, Object> getMappings(
100102
return merged.isEmpty() ? null : merged;
101103
});
102104
}
105+
103106
return mappings;
104107
}
105108

@@ -221,7 +224,7 @@ private static Map<String, Object> getMappingsForField(
221224
mappingForField.put(TYPE, ESUtils.LONG_FIELD_TYPE);
222225
} else if (fieldType == FieldType.DATETIME) {
223226
mappingForField.put(TYPE, ESUtils.DATE_FIELD_TYPE);
224-
} else if (fieldType == FieldType.OBJECT) {
227+
} else if (OBJECT_FIELD_TYPES.contains(fieldType)) {
225228
mappingForField.put(TYPE, ESUtils.OBJECT_FIELD_TYPE);
226229
} else if (fieldType == FieldType.DOUBLE) {
227230
mappingForField.put(TYPE, ESUtils.DOUBLE_FIELD_TYPE);

0 commit comments

Comments
 (0)