Skip to content

Commit 16a6830

Browse files
sgomezvillamorsgomezvillamor
authored
Merge branch 'master' into feature/cus-3105-fix-ge-profiler-try-except-timeout
2 parents a871995 + 86f4b80 commit 16a6830

File tree

11 files changed

+122
-43
lines changed

11 files changed

+122
-43
lines changed

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/incident/IncidentUtils.java

+5-3
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import com.linkedin.common.AuditStamp;
77
import com.linkedin.common.urn.Urn;
88
import com.linkedin.common.urn.UrnUtils;
9+
import com.linkedin.data.template.SetMode;
910
import com.linkedin.datahub.graphql.QueryContext;
1011
import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
1112
import com.linkedin.datahub.graphql.generated.IncidentPriority;
@@ -92,12 +93,13 @@ public static IncidentStatus mapIncidentStatus(
9293

9394
IncidentStatus status = new IncidentStatus();
9495
status.setState(IncidentState.valueOf(input.getState().toString()));
95-
if (input.getStage() != null) {
96-
status.setStage(IncidentStage.valueOf(input.getStage().toString()));
97-
}
96+
status.setStage(
97+
input.getStage() == null ? null : IncidentStage.valueOf(input.getStage().toString()),
98+
SetMode.REMOVE_IF_NULL);
9899
if (input.getMessage() != null) {
99100
status.setMessage(input.getMessage());
100101
}
102+
status.setLastUpdated(auditStamp);
101103
return status;
102104
}
103105

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/incident/UpdateIncidentResolver.java

+8-6
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import com.linkedin.common.UrnArray;
1313
import com.linkedin.common.urn.Urn;
1414
import com.linkedin.common.urn.UrnUtils;
15+
import com.linkedin.data.template.SetMode;
1516
import com.linkedin.datahub.graphql.QueryContext;
1617
import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
1718
import com.linkedin.datahub.graphql.exception.AuthorizationException;
@@ -116,18 +117,19 @@ private void updateIncidentInfo(
116117
if (input.getDescription() != null) {
117118
info.setDescription(input.getDescription());
118119
}
119-
if (input.getPriority() != null) {
120-
info.setPriority(IncidentUtils.mapIncidentPriority(input.getPriority()));
121-
}
122-
if (input.getAssigneeUrns() != null) {
123-
info.setAssignees(IncidentUtils.mapIncidentAssignees(input.getAssigneeUrns(), actorStamp));
124-
}
125120
if (input.getStatus() != null) {
126121
info.setStatus(IncidentUtils.mapIncidentStatus(input.getStatus(), actorStamp));
127122
}
128123
if (input.getResourceUrns() != null && !input.getResourceUrns().isEmpty()) {
129124
info.setEntities(new UrnArray(IncidentUtils.stringsToUrns(input.getResourceUrns())));
130125
}
126+
127+
info.setPriority(
128+
IncidentUtils.mapIncidentPriority(input.getPriority()), SetMode.REMOVE_IF_NULL);
129+
130+
info.setAssignees(
131+
IncidentUtils.mapIncidentAssignees(input.getAssigneeUrns(), actorStamp),
132+
SetMode.REMOVE_IF_NULL);
131133
}
132134

133135
private boolean isAuthorizedToUpdateIncident(final Urn resourceUrn, final QueryContext context) {

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryNodeType.java

+49-1
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,22 @@
1111
import com.linkedin.common.urn.Urn;
1212
import com.linkedin.common.urn.UrnUtils;
1313
import com.linkedin.datahub.graphql.QueryContext;
14+
import com.linkedin.datahub.graphql.generated.AutoCompleteResults;
1415
import com.linkedin.datahub.graphql.generated.Entity;
1516
import com.linkedin.datahub.graphql.generated.EntityType;
17+
import com.linkedin.datahub.graphql.generated.FacetFilterInput;
1618
import com.linkedin.datahub.graphql.generated.GlossaryNode;
19+
import com.linkedin.datahub.graphql.generated.SearchResults;
20+
import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
21+
import com.linkedin.datahub.graphql.types.SearchableEntityType;
1722
import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryNodeMapper;
23+
import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper;
24+
import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper;
1825
import com.linkedin.entity.EntityResponse;
1926
import com.linkedin.entity.client.EntityClient;
27+
import com.linkedin.metadata.query.AutoCompleteResult;
28+
import com.linkedin.metadata.query.filter.Filter;
29+
import com.linkedin.metadata.search.SearchResult;
2030
import graphql.execution.DataFetcherResult;
2131
import java.util.ArrayList;
2232
import java.util.HashSet;
@@ -25,9 +35,12 @@
2535
import java.util.Set;
2636
import java.util.function.Function;
2737
import java.util.stream.Collectors;
38+
import javax.annotation.Nonnull;
39+
import javax.annotation.Nullable;
2840

2941
public class GlossaryNodeType
30-
implements com.linkedin.datahub.graphql.types.EntityType<GlossaryNode, String> {
42+
implements SearchableEntityType<GlossaryNode, String>,
43+
com.linkedin.datahub.graphql.types.EntityType<GlossaryNode, String> {
3144

3245
static final Set<String> ASPECTS_TO_RESOLVE =
3346
ImmutableSet.of(
@@ -89,4 +102,39 @@ public List<DataFetcherResult<GlossaryNode>> batchLoad(
89102
throw new RuntimeException("Failed to batch load GlossaryNodes", e);
90103
}
91104
}
105+
106+
@Override
107+
public SearchResults search(
108+
@Nonnull String query,
109+
@Nullable List<FacetFilterInput> filters,
110+
int start,
111+
int count,
112+
@Nonnull final QueryContext context)
113+
throws Exception {
114+
final Map<String, String> facetFilters =
115+
ResolverUtils.buildFacetFilters(filters, ImmutableSet.of());
116+
final SearchResult searchResult =
117+
_entityClient.search(
118+
context.getOperationContext().withSearchFlags(flags -> flags.setFulltext(true)),
119+
GLOSSARY_NODE_ENTITY_NAME,
120+
query,
121+
facetFilters,
122+
start,
123+
count);
124+
return UrnSearchResultsMapper.map(context, searchResult);
125+
}
126+
127+
@Override
128+
public AutoCompleteResults autoComplete(
129+
@Nonnull String query,
130+
@Nullable String field,
131+
@Nullable Filter filters,
132+
int limit,
133+
@Nonnull final QueryContext context)
134+
throws Exception {
135+
final AutoCompleteResult result =
136+
_entityClient.autoComplete(
137+
context.getOperationContext(), GLOSSARY_NODE_ENTITY_NAME, query, filters, limit);
138+
return AutoCompleteResultsMapper.map(context, result);
139+
}
92140
}

‎datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/incident/UpdateIncidentResolverTest.java

+2-22
Original file line numberDiff line numberDiff line change
@@ -53,17 +53,7 @@ public void testGetSuccessAllFields() throws Exception {
5353
existingInfo.setEntities(
5454
new UrnArray(ImmutableList.of(UrnUtils.getUrn("urn:li:dataset:(test,test,test)"))));
5555
existingInfo.setStatus(
56-
new IncidentStatus()
57-
.setState(IncidentState.ACTIVE)
58-
.setStage(IncidentStage.INVESTIGATION)
59-
.setMessage("Message"));
60-
existingInfo.setAssignees(
61-
new IncidentAssigneeArray(
62-
ImmutableList.of(
63-
new IncidentAssignee()
64-
.setActor(UrnUtils.getUrn("urn:li:corpuser:test"))
65-
.setAssignedAt(new AuditStamp()))));
66-
existingInfo.setPriority(0);
56+
new IncidentStatus().setState(IncidentState.ACTIVE).setMessage("Message"));
6757
existingInfo.setSource(new IncidentSource().setType(IncidentSourceType.MANUAL));
6858

6959
EntityService mockEntityService = Mockito.mock(EntityService.class);
@@ -198,17 +188,7 @@ public void testGetSuccessRequiredFields() throws Exception {
198188
expectedInfo.setEntities(
199189
new UrnArray(ImmutableList.of(UrnUtils.getUrn("urn:li:dataset:(test,test,test)"))));
200190
expectedInfo.setStatus(
201-
new IncidentStatus()
202-
.setState(IncidentState.ACTIVE)
203-
.setStage(IncidentStage.INVESTIGATION)
204-
.setMessage("Message"));
205-
expectedInfo.setAssignees(
206-
new IncidentAssigneeArray(
207-
ImmutableList.of(
208-
new IncidentAssignee()
209-
.setActor(UrnUtils.getUrn("urn:li:corpuser:test"))
210-
.setAssignedAt(new AuditStamp()))));
211-
expectedInfo.setPriority(0);
191+
new IncidentStatus().setState(IncidentState.ACTIVE).setMessage("Message"));
212192
expectedInfo.setSource(new IncidentSource().setType(IncidentSourceType.MANUAL));
213193

214194
// Verify entity client

‎datahub-web-react/src/app/entity/shared/ExternalUrlButton.tsx

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import UrlButton from './UrlButton';
55

66
const GITHUB_LINK = 'github.com';
77
const GITHUB = 'GitHub';
8-
const GITLAB_LINK = 'gitlab.com';
8+
const GITLAB_LINK = 'gitlab.';
99
const GITLAB = 'GitLab';
1010

1111
interface Props {

‎datahub-web-react/src/app/entityV2/shared/utils.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ export function getExternalUrlDisplayName(entity: GenericEntityProperties | null
103103
// Scoping these constants
104104
const GITHUB_LINK = 'github.com';
105105
const GITHUB_NAME = 'GitHub';
106-
const GITLAB_LINK = 'gitlab.com';
106+
const GITLAB_LINK = 'gitlab.';
107107
const GITLAB_NAME = 'GitLab';
108108

109109
const externalUrl = entity?.properties?.externalUrl;

‎metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py

+1
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,7 @@ def _process_table(
246246
platform=self.platform,
247247
platform_instance=platform_instance,
248248
name=dataset_name,
249+
env=self.config.env,
249250
)
250251
dataset_properties = DatasetPropertiesClass(
251252
name=table_name,

‎metadata-ingestion/src/datahub/ingestion/source/mlflow.py

+7-1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
DataProcessInstanceRunEventClass,
4848
DataProcessInstanceRunResultClass,
4949
DataProcessRunStatusClass,
50+
EdgeClass,
5051
GlobalTagsClass,
5152
MetadataAttributionClass,
5253
MLHyperParamClass,
@@ -312,7 +313,12 @@ def _get_run_workunits(
312313
model_version_urn = self._make_ml_model_urn(model_versions[0])
313314
yield MetadataChangeProposalWrapper(
314315
entityUrn=str(data_process_instance.urn),
315-
aspect=DataProcessInstanceOutputClass(outputs=[model_version_urn]),
316+
aspect=DataProcessInstanceOutputClass(
317+
outputs=[],
318+
outputEdges=[
319+
EdgeClass(destinationUrn=model_version_urn),
320+
],
321+
),
316322
).as_workunit()
317323

318324
metrics = self._get_run_metrics(run)

‎metadata-ingestion/tests/integration/mlflow/mlflow_mcps_golden.json

+5-2
Original file line numberDiff line numberDiff line change
@@ -308,8 +308,11 @@
308308
"aspectName": "dataProcessInstanceOutput",
309309
"aspect": {
310310
"json": {
311-
"outputs": [
312-
"urn:li:mlModel:(urn:li:dataPlatform:mlflow,test-model_1,PROD)"
311+
"outputs": [],
312+
"outputEdges": [
313+
{
314+
"destinationUrn": "urn:li:mlModel:(urn:li:dataPlatform:mlflow,test-model_1,PROD)"
315+
}
313316
]
314317
}
315318
},

‎metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceInput.pdl

+21-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
namespace com.linkedin.dataprocess
22

3+
import com.linkedin.common.Edge
34
import com.linkedin.common.Urn
45

56
/**
@@ -15,8 +16,7 @@ record DataProcessInstanceInput {
1516
@Relationship = {
1617
"/*": {
1718
"name": "Consumes",
18-
"entityTypes": [ "dataset", "mlModel"],
19-
"isLineage": true
19+
"entityTypes": [ "dataset", "mlModel" ]
2020
}
2121
}
2222
@Searchable = {
@@ -29,4 +29,23 @@ record DataProcessInstanceInput {
2929
}
3030
}
3131
inputs: array[Urn]
32+
33+
/**
34+
* Input assets consumed by the data process instance, with additional metadata.
35+
* Counts as lineage.
36+
* Will eventually deprecate the inputs field.
37+
*/
38+
@Relationship = {
39+
"/*/destinationUrn": {
40+
"name": "DataProcessInstanceConsumes",
41+
"entityTypes": [ "dataset", "mlModel" ],
42+
"isLineage": true,
43+
"createdOn": "inputEdges/*/created/time"
44+
"createdActor": "inputEdges/*/created/actor"
45+
"updatedOn": "inputEdges/*/lastModified/time"
46+
"updatedActor": "inputEdges/*/lastModified/actor"
47+
"properties": "inputEdges/*/properties"
48+
}
49+
}
50+
inputEdges: optional array[Edge]
3251
}

‎metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceOutput.pdl

+22-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
namespace com.linkedin.dataprocess
22

3+
import com.linkedin.common.Edge
34
import com.linkedin.common.Urn
45

56
/**
@@ -10,14 +11,12 @@ import com.linkedin.common.Urn
1011
}
1112
record DataProcessInstanceOutput {
1213
/**
13-
* Output datasets to be produced
14+
* Output assets produced
1415
*/
1516
@Relationship = {
1617
"/*": {
1718
"name": "Produces",
18-
"entityTypes": [ "dataset", "mlModel" ],
19-
"isLineage": true,
20-
"isUpstream": false
19+
"entityTypes": [ "dataset", "mlModel" ]
2120
}
2221
}
2322
@Searchable = {
@@ -31,4 +30,23 @@ record DataProcessInstanceOutput {
3130
}
3231
outputs: array[Urn]
3332

33+
/**
34+
* Output assets produced by the data process instance during processing, with additional metadata.
35+
* Counts as lineage.
36+
* Will eventually deprecate the outputs field.
37+
*/
38+
@Relationship = {
39+
"/*/destinationUrn": {
40+
"name": "DataProcessInstanceProduces",
41+
"entityTypes": [ "dataset", "mlModel" ],
42+
"isUpstream": false,
43+
"isLineage": true,
44+
"createdOn": "outputEdges/*/created/time"
45+
"createdActor": "outputEdges/*/created/actor"
46+
"updatedOn": "outputEdges/*/lastModified/time"
47+
"updatedActor": "outputEdges/*/lastModified/actor"
48+
"properties": "outputEdges/*/properties"
49+
}
50+
}
51+
outputEdges: optional array[Edge]
3452
}

0 commit comments

Comments
 (0)