Skip to content

Commit fe5d3b7

Browse files
david-leifkerdavid-leifker
authored
feat(graphql): implement sort and facet for scroll (#12746)
1 parent 61d4992 commit fe5d3b7

File tree

36 files changed

+519
-205
lines changed

36 files changed

+519
-205
lines changed

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetMetadataAnalyticsResolver.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ private List<AnalyticsChart> getCharts(MetadataAnalyticsInput input, OperationCo
7878

7979
SearchResult searchResult =
8080
_entityClient.searchAcrossEntities(
81-
opContext, entities, query, filter, 0, 0, Collections.emptyList(), null);
81+
opContext, entities, query, filter, 0, 0, Collections.emptyList());
8282

8383
List<AggregationMetadata> aggregationMetadataList =
8484
searchResult.getMetadata().getAggregations();

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/container/ContainerEntitiesResolver.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,7 @@ public CompletableFuture<SearchResults> get(final DataFetchingEnvironment enviro
9191
new CriterionArray(ImmutableList.of(filterCriterion))))),
9292
start,
9393
count,
94-
Collections.emptyList(),
95-
null));
94+
Collections.emptyList()));
9695

9796
} catch (Exception e) {
9897
throw new RuntimeException(

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java

-1
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,6 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
185185
finalFilter,
186186
start,
187187
count,
188-
null,
189188
null));
190189
results
191190
.getSearchResults()

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,7 @@ public CompletableFuture<SearchResults> get(final DataFetchingEnvironment enviro
9595
new ConjunctiveCriterion().setAnd(criteria))),
9696
start,
9797
count,
98-
Collections.emptyList(),
99-
null));
98+
Collections.emptyList()));
10099

101100
} catch (Exception e) {
102101
throw new RuntimeException(

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolver.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,7 @@ private SearchResult getSearchResults(
108108
: null,
109109
0,
110110
0,
111-
Collections.emptyList(),
112-
null);
111+
Collections.emptyList());
113112
}
114113

115114
/**

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java

+3
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import com.linkedin.entity.client.EntityClient;
1717
import com.linkedin.metadata.query.SearchFlags;
1818
import com.linkedin.metadata.query.filter.Filter;
19+
import com.linkedin.metadata.query.filter.SortCriterion;
1920
import com.linkedin.metadata.service.ViewService;
2021
import com.linkedin.view.DataHubViewInfo;
2122
import graphql.schema.DataFetcher;
@@ -80,6 +81,7 @@ public CompletableFuture<ScrollResults> get(DataFetchingEnvironment environment)
8081
} else {
8182
searchFlags = null;
8283
}
84+
List<SortCriterion> sortCriteria = SearchUtils.getSortCriteria(input.getSortInput());
8385

8486
try {
8587
log.debug(
@@ -108,6 +110,7 @@ public CompletableFuture<ScrollResults> get(DataFetchingEnvironment environment)
108110
: baseFilter,
109111
scrollId,
110112
keepAlive,
113+
sortCriteria,
111114
count));
112115
} catch (Exception e) {
113116
log.error(

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -146,8 +146,7 @@ private List<String> getStructuredPropertyFacets(final QueryContext context) {
146146
createStructuredPropertyFilter(),
147147
0,
148148
100,
149-
Collections.emptyList(),
150-
null);
149+
Collections.emptyList());
151150
return result.getEntities().stream()
152151
.map(entity -> String.format("structuredProperties.%s", entity.getEntity().getId()))
153152
.collect(Collectors.toList());

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java

+12-9
Original file line numberDiff line numberDiff line change
@@ -378,6 +378,8 @@ public static CompletableFuture<ScrollResults> scrollAcrossEntities(
378378
Integer inputCount,
379379
String scrollId,
380380
String inputKeepAlive,
381+
List<SortCriterion> sortCriteria,
382+
List<String> facets,
381383
String className) {
382384

383385
final List<EntityType> entityTypes =
@@ -431,7 +433,15 @@ public static CompletableFuture<ScrollResults> scrollAcrossEntities(
431433
try {
432434
final ScrollResult scrollResult =
433435
_entityClient.scrollAcrossEntities(
434-
context, finalEntityNames, query, finalFilters, scrollId, keepAlive, count);
436+
context,
437+
finalEntityNames,
438+
query,
439+
finalFilters,
440+
scrollId,
441+
keepAlive,
442+
sortCriteria,
443+
count,
444+
facets);
435445
return UrnScrollResultsMapper.map(inputContext, scrollResult);
436446
} catch (Exception e) {
437447
log.warn(
@@ -518,14 +528,7 @@ public static CompletableFuture<SearchResults> searchAcrossEntities(
518528
try {
519529
final SearchResult searchResult =
520530
_entityClient.searchAcrossEntities(
521-
context,
522-
finalEntityNames,
523-
query,
524-
finalFilters,
525-
start,
526-
count,
527-
sortCriteria,
528-
null);
531+
context, finalEntityNames, query, finalFilters, start, count, sortCriteria);
529532
return UrnSearchResultsMapper.map(inputContext, searchResult);
530533
} catch (Exception e) {
531534
log.warn(

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/siblings/SiblingsSearchResolver.java

+3
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import com.linkedin.metadata.utils.CriterionUtils;
2020
import graphql.schema.DataFetcher;
2121
import graphql.schema.DataFetchingEnvironment;
22+
import java.util.List;
2223
import java.util.concurrent.CompletableFuture;
2324
import lombok.RequiredArgsConstructor;
2425
import lombok.extern.slf4j.Slf4j;
@@ -61,6 +62,8 @@ public CompletableFuture<ScrollResults> get(DataFetchingEnvironment environment)
6162
input.getCount(),
6263
input.getScrollId(),
6364
input.getKeepAlive(),
65+
List.of(),
66+
List.of(),
6467
this.getClass().getSimpleName());
6568
}
6669
}

‎datahub-graphql-core/src/main/resources/search.graphql

+5
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,11 @@ input ScrollAcrossEntitiesInput {
347347
Flags controlling search options
348348
"""
349349
searchFlags: SearchFlags
350+
351+
"""
352+
Optional - Information on how to sort this search result
353+
"""
354+
sortInput: SearchSortInput
350355
}
351356

352357

‎datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/container/ContainerEntitiesResolverTest.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,7 @@ public void testGetSuccess() throws Exception {
6060
new CriterionArray(ImmutableList.of(filterCriterion)))))),
6161
Mockito.eq(0),
6262
Mockito.eq(20),
63-
Mockito.eq(Collections.emptyList()),
64-
Mockito.eq(null)))
63+
Mockito.eq(Collections.emptyList())))
6564
.thenReturn(
6665
new SearchResult()
6766
.setFrom(0)

‎datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolverTest.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,7 @@ public void testGetSuccess() throws Exception {
6565
new CriterionArray(ImmutableList.of(filterCriterion)))))),
6666
Mockito.eq(0),
6767
Mockito.eq(20),
68-
Mockito.eq(Collections.emptyList()),
69-
Mockito.eq(null)))
68+
Mockito.eq(Collections.emptyList())))
7069
.thenReturn(
7170
new SearchResult()
7271
.setFrom(0)

‎datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolverTest.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -301,8 +301,7 @@ private static EntityClient initMockEntityClient(
301301
Mockito.eq(filter),
302302
Mockito.eq(start),
303303
Mockito.eq(limit),
304-
Mockito.eq(Collections.emptyList()),
305-
Mockito.eq(null)))
304+
Mockito.eq(Collections.emptyList())))
306305
.thenReturn(result);
307306
return client;
308307
}

‎datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/versioning/VersionsSearchResolverTest.java

+28-15
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import static com.linkedin.metadata.Constants.*;
55
import static com.linkedin.metadata.utils.CriterionUtils.*;
66
import static org.mockito.ArgumentMatchers.*;
7+
import static org.mockito.ArgumentMatchers.anyString;
78
import static org.testng.Assert.assertEquals;
89
import static org.testng.Assert.assertThrows;
910

@@ -39,6 +40,7 @@
3940
import com.linkedin.view.DataHubViewInfo;
4041
import com.linkedin.view.DataHubViewType;
4142
import graphql.schema.DataFetchingEnvironment;
43+
import io.datahubproject.metadata.context.OperationContext;
4244
import java.util.List;
4345
import java.util.concurrent.CompletionException;
4446
import org.mockito.Mockito;
@@ -144,8 +146,7 @@ public void testGetSuccessBasic() throws Exception {
144146
List.of(
145147
new com.linkedin.metadata.query.filter.SortCriterion()
146148
.setField(VERSION_SORT_ID_FIELD_NAME)
147-
.setOrder(com.linkedin.metadata.query.filter.SortOrder.DESCENDING))),
148-
any());
149+
.setOrder(com.linkedin.metadata.query.filter.SortOrder.DESCENDING))));
149150
}
150151

151152
@Test
@@ -240,8 +241,7 @@ public void testGetSuccessComplex() throws Exception {
240241
.setOrder(com.linkedin.metadata.query.filter.SortOrder.ASCENDING),
241242
new com.linkedin.metadata.query.filter.SortCriterion()
242243
.setField(VERSION_SORT_ID_FIELD_NAME)
243-
.setOrder(com.linkedin.metadata.query.filter.SortOrder.DESCENDING))),
244-
any());
244+
.setOrder(com.linkedin.metadata.query.filter.SortOrder.DESCENDING))));
245245
}
246246

247247
@Test
@@ -251,7 +251,7 @@ public void testThrowsError() throws Exception {
251251

252252
Mockito.when(
253253
mockEntityClient.searchAcrossEntities(
254-
any(), any(), any(), any(), Mockito.anyInt(), Mockito.anyInt(), any(), any()))
254+
any(), any(), any(), any(), Mockito.anyInt(), Mockito.anyInt(), any()))
255255
.thenThrow(new RemoteInvocationException());
256256

257257
VersionsSearchResolver resolver = new VersionsSearchResolver(mockEntityClient, mockViewService);
@@ -271,23 +271,36 @@ public void testThrowsError() throws Exception {
271271
private EntityClient initMockEntityClient() throws Exception {
272272
EntityClient client = Mockito.mock(EntityClient.class);
273273

274+
SearchResult result =
275+
new SearchResult()
276+
.setEntities(new SearchEntityArray())
277+
.setNumEntities(0)
278+
.setFrom(0)
279+
.setPageSize(0)
280+
.setMetadata(new SearchResultMetadata());
281+
274282
Mockito.when(
275283
client.searchAcrossEntities(
276-
any(),
284+
any(OperationContext.class),
277285
any(),
278286
Mockito.anyString(),
279-
any(),
287+
any(Filter.class),
280288
Mockito.anyInt(),
281289
Mockito.anyInt(),
290+
anyList(),
291+
anyList()))
292+
.thenReturn(result);
293+
294+
Mockito.when(
295+
client.searchAcrossEntities(
296+
any(OperationContext.class),
282297
any(),
283-
Mockito.eq(null)))
284-
.thenReturn(
285-
new SearchResult()
286-
.setEntities(new SearchEntityArray())
287-
.setNumEntities(0)
288-
.setFrom(0)
289-
.setPageSize(0)
290-
.setMetadata(new SearchResultMetadata()));
298+
anyString(),
299+
any(Filter.class),
300+
anyInt(),
301+
anyInt(),
302+
anyList()))
303+
.thenReturn(result);
291304

292305
return client;
293306
}

‎docs/api/graphql/graphql-best-practices.md

+40-4
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,14 @@ This technique makes maintaining your GraphQL queries much more doable. For exam
3333

3434
`search*` APIs such as [`searchAcrossEntities`](https://datahubproject.io/docs/GraphQL/queries/#searchacrossentities) are designed for minimal pagination (< ~50). They do not perform well for deep pagination requests. Use the equivalent `scroll*` APIs such as [`scrollAcrossEntities`](https://datahubproject.io/docs/GraphQL/queries/#scrollacrossentities) when expecting the need to paginate deeply into the result set.
3535

36-
Note: that it is impossible to use `search*` for paginating beyond 10k results.
36+
:::note
37+
It is impossible to use `search*` for paginating beyond 10k results.
38+
:::
39+
40+
:::caution
41+
In order to `scroll*` through the entire result set it is required to use a stable sort order. This means using `_score` as
42+
the first sort order cannot be used. Use the `urn` field as the sort order instead.
43+
:::
3744

3845
#### Examples
3946

@@ -54,7 +61,15 @@ Page 1 Request:
5461
orFilters: [
5562
{ and: [{ field: "name", condition: CONTAIN, values: ["pet"] }] },
5663
{ and: [{ field: "title", condition: CONTAIN, values: ["pet"] }] }
57-
]
64+
],
65+
sortInput: {
66+
sortCriteria: [
67+
{
68+
field: "urn",
69+
sortOrder: ASCENDING
70+
}
71+
]
72+
}
5873
}
5974
) {
6075
nextScrollId
@@ -110,7 +125,15 @@ Page 2 Request:
110125
orFilters: [
111126
{ and: [{ field: "name", condition: CONTAIN, values: ["pet"] }] },
112127
{ and: [{ field: "title", condition: CONTAIN, values: ["pet"] }] }
113-
]
128+
],
129+
sortInput: {
130+
sortCriteria: [
131+
{
132+
field: "urn",
133+
sortOrder: ASCENDING
134+
}
135+
]
136+
}
114137
}
115138
) {
116139
nextScrollId
@@ -282,7 +305,20 @@ Example for skipping highlighting and aggregates, typically used for scrolling s
282305
```graphql
283306
{
284307
scrollAcrossEntities(
285-
input: {types: [DATASET], count: 2, query: "pet", searchFlags: {skipAggregates: true, skipHighlighting: true}}
308+
input: {
309+
types: [DATASET],
310+
count: 2,
311+
query: "pet",
312+
searchFlags: {skipAggregates: true, skipHighlighting: true},
313+
sortInput: {
314+
sortCriteria: [
315+
{
316+
field: "urn",
317+
sortOrder: ASCENDING
318+
}
319+
]
320+
},
321+
}
286322
) {
287323
searchResults {
288324
entity {

0 commit comments

Comments
 (0)