Skip to content

Commit c6fd3bc

Browse files
gabe-lyonsgabe-lyons
committed
adding dataTransformLogic models
1 parent acb76cd commit c6fd3bc

File tree

12 files changed

+424
-58
lines changed

12 files changed

+424
-58
lines changed

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DataTransformLogicMapper.java

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package com.linkedin.datahub.graphql.types.common.mappers;
2+
3+
import com.linkedin.datahub.graphql.QueryContext;
4+
import com.linkedin.datahub.graphql.generated.DataTransform;
5+
import com.linkedin.datahub.graphql.generated.DataTransformLogic;
6+
import com.linkedin.datahub.graphql.generated.QueryLanguage;
7+
import com.linkedin.datahub.graphql.generated.QueryStatement;
8+
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
9+
import java.util.stream.Collectors;
10+
import javax.annotation.Nonnull;
11+
import javax.annotation.Nullable;
12+
13+
public class DataTransformLogicMapper
14+
implements ModelMapper<
15+
com.linkedin.common.DataTransformLogic,
16+
com.linkedin.datahub.graphql.generated.DataTransformLogic> {
17+
18+
public static final DataTransformLogicMapper INSTANCE = new DataTransformLogicMapper();
19+
20+
public static DataTransformLogic map(
21+
@Nullable final QueryContext context,
22+
@Nonnull final com.linkedin.common.DataTransformLogic input) {
23+
return INSTANCE.apply(context, input);
24+
}
25+
26+
@Override
27+
public DataTransformLogic apply(
28+
@Nullable final QueryContext context,
29+
@Nonnull final com.linkedin.common.DataTransformLogic input) {
30+
31+
final DataTransformLogic result = new DataTransformLogic();
32+
33+
// Map transforms array using DataTransformMapper
34+
result.setTransforms(
35+
input.getTransforms().stream()
36+
.map(transform -> DataTransformMapper.map(context, transform))
37+
.collect(Collectors.toList()));
38+
39+
return result;
40+
}
41+
}
42+
43+
class DataTransformMapper
44+
implements ModelMapper<
45+
com.linkedin.common.DataTransform, com.linkedin.datahub.graphql.generated.DataTransform> {
46+
47+
public static final DataTransformMapper INSTANCE = new DataTransformMapper();
48+
49+
public static DataTransform map(
50+
@Nullable final QueryContext context,
51+
@Nonnull final com.linkedin.common.DataTransform input) {
52+
return INSTANCE.apply(context, input);
53+
}
54+
55+
@Override
56+
public DataTransform apply(
57+
@Nullable final QueryContext context,
58+
@Nonnull final com.linkedin.common.DataTransform input) {
59+
60+
final DataTransform result = new DataTransform();
61+
62+
// Map query statement if present
63+
if (input.hasQueryStatement()) {
64+
QueryStatement statement =
65+
new QueryStatement(
66+
input.getQueryStatement().getValue(),
67+
QueryLanguage.valueOf(input.getQueryStatement().getLanguage().toString()));
68+
result.setQueryStatement(statement);
69+
}
70+
71+
return result;
72+
}
73+
}

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/QueryPropertiesMapper.java

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package com.linkedin.datahub.graphql.types.common.mappers;
2+
3+
import com.linkedin.data.template.GetMode;
4+
import com.linkedin.datahub.graphql.QueryContext;
5+
import com.linkedin.datahub.graphql.generated.*;
6+
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
7+
import com.linkedin.query.QueryProperties;
8+
import javax.annotation.Nonnull;
9+
import javax.annotation.Nullable;
10+
11+
public class QueryPropertiesMapper
12+
implements ModelMapper<
13+
QueryProperties, com.linkedin.datahub.graphql.generated.QueryProperties> {
14+
15+
public static final QueryPropertiesMapper INSTANCE = new QueryPropertiesMapper();
16+
17+
public static com.linkedin.datahub.graphql.generated.QueryProperties map(
18+
@Nullable final QueryContext context, @Nonnull final QueryProperties input) {
19+
return INSTANCE.apply(context, input);
20+
}
21+
22+
@Override
23+
public com.linkedin.datahub.graphql.generated.QueryProperties apply(
24+
@Nullable final QueryContext context, @Nonnull final QueryProperties input) {
25+
26+
final com.linkedin.datahub.graphql.generated.QueryProperties result =
27+
new com.linkedin.datahub.graphql.generated.QueryProperties();
28+
29+
// Map Query Source
30+
result.setSource(QuerySource.valueOf(input.getSource().toString()));
31+
32+
// Map Query Statement
33+
result.setStatement(
34+
new QueryStatement(
35+
input.getStatement().getValue(),
36+
QueryLanguage.valueOf(input.getStatement().getLanguage().toString())));
37+
38+
// Map optional fields
39+
result.setName(input.getName(GetMode.NULL));
40+
result.setDescription(input.getDescription(GetMode.NULL));
41+
42+
// Map origin if present
43+
if (input.hasOrigin() && input.getOrigin() != null) {
44+
result.setOrigin(UrnToEntityMapper.map(context, input.getOrigin()));
45+
}
46+
47+
// Map created audit stamp
48+
AuditStamp created = new AuditStamp();
49+
created.setTime(input.getCreated().getTime());
50+
created.setActor(input.getCreated().getActor(GetMode.NULL).toString());
51+
result.setCreated(created);
52+
53+
// Map last modified audit stamp
54+
AuditStamp lastModified = new AuditStamp();
55+
lastModified.setTime(input.getLastModified().getTime());
56+
lastModified.setActor(input.getLastModified().getActor(GetMode.NULL).toString());
57+
result.setLastModified(lastModified);
58+
59+
return result;
60+
}
61+
}

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,8 @@ public class DataJobType
7979
BROWSE_PATHS_V2_ASPECT_NAME,
8080
SUB_TYPES_ASPECT_NAME,
8181
STRUCTURED_PROPERTIES_ASPECT_NAME,
82-
FORMS_ASPECT_NAME);
82+
FORMS_ASPECT_NAME,
83+
DATA_TRANSFORM_LOGIC_ASPECT_NAME);
8384
private static final Set<String> FACET_FIELDS = ImmutableSet.of("flow");
8485
private final EntityClient _entityClient;
8586

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java

+5-19
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,7 @@
44
import static com.linkedin.metadata.Constants.*;
55

66
import com.google.common.collect.ImmutableList;
7-
import com.linkedin.common.BrowsePathsV2;
8-
import com.linkedin.common.DataPlatformInstance;
9-
import com.linkedin.common.Deprecation;
10-
import com.linkedin.common.Forms;
11-
import com.linkedin.common.GlobalTags;
12-
import com.linkedin.common.GlossaryTerms;
13-
import com.linkedin.common.InstitutionalMemory;
14-
import com.linkedin.common.Ownership;
15-
import com.linkedin.common.Status;
16-
import com.linkedin.common.SubTypes;
7+
import com.linkedin.common.*;
178
import com.linkedin.common.urn.Urn;
189
import com.linkedin.data.DataMap;
1910
import com.linkedin.datahub.graphql.QueryContext;
@@ -26,15 +17,7 @@
2617
import com.linkedin.datahub.graphql.generated.DataJobProperties;
2718
import com.linkedin.datahub.graphql.generated.Dataset;
2819
import com.linkedin.datahub.graphql.generated.EntityType;
29-
import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper;
30-
import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
31-
import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper;
32-
import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper;
33-
import com.linkedin.datahub.graphql.types.common.mappers.FineGrainedLineagesMapper;
34-
import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper;
35-
import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper;
36-
import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
37-
import com.linkedin.datahub.graphql.types.common.mappers.SubTypesMapper;
20+
import com.linkedin.datahub.graphql.types.common.mappers.*;
3821
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
3922
import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
4023
import com.linkedin.datahub.graphql.types.form.FormsMapper;
@@ -139,6 +122,9 @@ public DataJob apply(
139122
context, new StructuredProperties(data), entityUrn));
140123
} else if (FORMS_ASPECT_NAME.equals(name)) {
141124
result.setForms(FormsMapper.map(new Forms(data), entityUrn.toString()));
125+
} else if (DATA_TRANSFORM_LOGIC_ASPECT_NAME.equals(name)) {
126+
result.setDataTransformLogic(
127+
DataTransformLogicMapper.map(context, new DataTransformLogic(data)));
142128
}
143129
});
144130

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/query/QueryMapper.java

+5-38
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,13 @@
55
import com.linkedin.common.DataPlatformInstance;
66
import com.linkedin.common.urn.Urn;
77
import com.linkedin.data.DataMap;
8-
import com.linkedin.data.template.GetMode;
98
import com.linkedin.datahub.graphql.QueryContext;
10-
import com.linkedin.datahub.graphql.generated.AuditStamp;
119
import com.linkedin.datahub.graphql.generated.DataPlatform;
1210
import com.linkedin.datahub.graphql.generated.Dataset;
1311
import com.linkedin.datahub.graphql.generated.EntityType;
1412
import com.linkedin.datahub.graphql.generated.QueryEntity;
15-
import com.linkedin.datahub.graphql.generated.QueryLanguage;
16-
import com.linkedin.datahub.graphql.generated.QuerySource;
17-
import com.linkedin.datahub.graphql.generated.QueryStatement;
1813
import com.linkedin.datahub.graphql.generated.QuerySubject;
19-
import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
14+
import com.linkedin.datahub.graphql.types.common.mappers.QueryPropertiesMapper;
2015
import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
2116
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
2217
import com.linkedin.entity.EntityResponse;
@@ -48,7 +43,10 @@ public QueryEntity apply(
4843
result.setType(EntityType.QUERY);
4944
EnvelopedAspectMap aspectMap = entityResponse.getAspects();
5045
MappingHelper<QueryEntity> mappingHelper = new MappingHelper<>(aspectMap, result);
51-
mappingHelper.mapToResult(context, QUERY_PROPERTIES_ASPECT_NAME, this::mapQueryProperties);
46+
mappingHelper.mapToResult(
47+
QUERY_PROPERTIES_ASPECT_NAME,
48+
(entity, dataMap) ->
49+
entity.setProperties(QueryPropertiesMapper.map(context, new QueryProperties(dataMap))));
5250
mappingHelper.mapToResult(QUERY_SUBJECTS_ASPECT_NAME, this::mapQuerySubjects);
5351
mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, this::mapPlatform);
5452
return mappingHelper.getResult();
@@ -64,37 +62,6 @@ private void mapPlatform(@Nonnull QueryEntity query, @Nonnull DataMap dataMap) {
6462
}
6563
}
6664

67-
private void mapQueryProperties(
68-
@Nullable final QueryContext context, @Nonnull QueryEntity query, @Nonnull DataMap dataMap) {
69-
QueryProperties queryProperties = new QueryProperties(dataMap);
70-
com.linkedin.datahub.graphql.generated.QueryProperties res =
71-
new com.linkedin.datahub.graphql.generated.QueryProperties();
72-
73-
// Query Source must be kept in sync.
74-
res.setSource(QuerySource.valueOf(queryProperties.getSource().toString()));
75-
res.setStatement(
76-
new QueryStatement(
77-
queryProperties.getStatement().getValue(),
78-
QueryLanguage.valueOf(queryProperties.getStatement().getLanguage().toString())));
79-
res.setName(queryProperties.getName(GetMode.NULL));
80-
res.setDescription(queryProperties.getDescription(GetMode.NULL));
81-
if (queryProperties.hasOrigin() && queryProperties.getOrigin() != null) {
82-
res.setOrigin(UrnToEntityMapper.map(context, queryProperties.getOrigin()));
83-
}
84-
85-
AuditStamp created = new AuditStamp();
86-
created.setTime(queryProperties.getCreated().getTime());
87-
created.setActor(queryProperties.getCreated().getActor(GetMode.NULL).toString());
88-
res.setCreated(created);
89-
90-
AuditStamp lastModified = new AuditStamp();
91-
lastModified.setTime(queryProperties.getLastModified().getTime());
92-
lastModified.setActor(queryProperties.getLastModified().getActor(GetMode.NULL).toString());
93-
res.setLastModified(lastModified);
94-
95-
query.setProperties(res);
96-
}
97-
9865
@Nonnull
9966
private void mapQuerySubjects(@Nonnull QueryEntity query, @Nonnull DataMap dataMap) {
10067
QuerySubjects querySubjects = new QuerySubjects(dataMap);

‎datahub-graphql-core/src/main/resources/entity.graphql

+25
Original file line numberDiff line numberDiff line change
@@ -6556,6 +6556,11 @@ type DataJob implements EntityWithRelationships & Entity & BrowsableEntity {
65566556
The forms associated with the Dataset
65576557
"""
65586558
forms: Forms
6559+
6560+
"""
6561+
Data Transform Logic associated with the Data Job
6562+
"""
6563+
dataTransformLogic: DataTransformLogic
65596564
}
65606565

65616566
"""
@@ -6773,6 +6778,26 @@ type DataJobInputOutput {
67736778
fineGrainedLineages: [FineGrainedLineage!]
67746779
}
67756780

6781+
"""
6782+
Information about a transformation applied to data assets
6783+
"""
6784+
type DataTransform {
6785+
"""
6786+
The transformation may be defined by a query statement
6787+
"""
6788+
queryStatement: QueryStatement
6789+
}
6790+
6791+
"""
6792+
Information about transformations applied to data assets
6793+
"""
6794+
type DataTransformLogic {
6795+
"""
6796+
List of transformations applied
6797+
"""
6798+
transforms: [DataTransform!]!
6799+
}
6800+
67766801
"""
67776802
Information about individual user usage of a Dataset
67786803
"""

0 commit comments

Comments
 (0)