Skip to content

Commit e72e425

Browse files
acrylJonnyacrylJonny
authored
Merge branch 'master' into hive-improvements
2 parents 44cc8e0 + eef2077 commit e72e425

File tree

570 files changed

+30919
-6701
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

570 files changed

+30919
-6701
lines changed

‎.github/workflows/build-and-test.yml

+4
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@ jobs:
7575
path: |
7676
~/.cache/uv
7777
key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
78+
- name: Install dependencies
79+
run: ./metadata-ingestion/scripts/install_deps.sh
7880
- name: Set up JDK 17
7981
uses: actions/setup-java@v4
8082
with:
@@ -83,6 +85,7 @@ jobs:
8385
- uses: gradle/actions/setup-gradle@v3
8486
- name: Gradle build (and test) for NOT metadata ingestion
8587
if: ${{ matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
88+
# datahub-schematron:cli excluded due to dependency on metadata-ingestion
8689
run: |
8790
./gradlew build \
8891
-x :metadata-ingestion:build \
@@ -100,6 +103,7 @@ jobs:
100103
-x :metadata-ingestion-modules:gx-plugin:check \
101104
-x :datahub-frontend:build \
102105
-x :datahub-web-react:build \
106+
-x :metadata-integration:java:datahub-schematron:cli:test \
103107
--parallel
104108
- name: Gradle build (and test) for frontend
105109
if: ${{ matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }}

‎.github/workflows/check-datahub-jars.yml

+1
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,5 @@ jobs:
4040
- name: check ${{ matrix.command }} jar
4141
run: |
4242
./gradlew :metadata-integration:java:${{ matrix.command }}:build --info
43+
./gradlew :metadata-integration:java:${{ matrix.command }}:checkShadowJar
4344
./gradlew :metadata-integration:java:${{ matrix.command }}:javadoc

‎.github/workflows/pr-labeler.yml

+5-4
Original file line numberDiff line numberDiff line change
@@ -29,23 +29,24 @@ jobs:
2929
"swaroopjagadish",
3030
"treff7es",
3131
"yoonhyejin",
32-
"eboneil",
3332
"gabe-lyons",
3433
"hsheth2",
3534
"jjoyce0510",
3635
"maggiehays",
3736
"pedro93",
3837
"RyanHolstien",
3938
"sakethvarma397",
40-
"Kunal-kankriya",
4139
"purnimagarg1",
42-
"dushayntAW",
4340
"sagar-salvi-apptware",
4441
"kushagra-apptware",
4542
"Salman-Apptware",
4643
"mayurinehate",
4744
"noggi",
48-
"skrydal"
45+
"skrydal",
46+
"kevinkarchacryl",
47+
"sgomezvillamor",
48+
"acrylJonny",
49+
"chakru-r"
4950
]'),
5051
github.actor
5152
)

‎build.gradle

+12-3
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ buildscript {
4848
// see also datahub-frontend/play.gradle
4949
ext.playVersion = '2.8.22'
5050
ext.playScalaVersion = '2.13'
51+
ext.akkaVersion = '2.6.21' // 2.7.0+ has incompatible license
5152
ext.log4jVersion = '2.23.1'
5253
ext.slf4jVersion = '1.7.36'
5354
ext.logbackClassic = '1.4.14'
@@ -56,7 +57,7 @@ buildscript {
5657
ext.hazelcastVersion = '5.3.6'
5758
ext.ebeanVersion = '15.5.2'
5859
ext.googleJavaFormatVersion = '1.18.1'
59-
ext.openLineageVersion = '1.19.0'
60+
ext.openLineageVersion = '1.24.2'
6061
ext.logbackClassicJava8 = '1.2.12'
6162

6263
ext.docker_registry = 'acryldata'
@@ -105,7 +106,14 @@ project.ext.spec = [
105106
]
106107

107108
project.ext.externalDependency = [
108-
'akkaHttp': "com.typesafe.akka:akka-http-core_$playScalaVersion:10.2.10",
109+
'akkaHttp': "com.typesafe.akka:akka-http-core_$playScalaVersion:10.2.10", // max version due to licensing
110+
'akkaActor': "com.typesafe.akka:akka-actor_$playScalaVersion:$akkaVersion",
111+
'akkaStream': "com.typesafe.akka:akka-stream_$playScalaVersion:$akkaVersion",
112+
'akkaActorTyped': "com.typesafe.akka:akka-actor-typed_$playScalaVersion:$akkaVersion",
113+
'akkaSlf4j': "com.typesafe.akka:akka-slf4j_$playScalaVersion:$akkaVersion",
114+
'akkaJackson': "com.typesafe.akka:akka-serialization-jackson_$playScalaVersion:$akkaVersion",
115+
'akkaParsing': "com.typesafe.akka:akka-parsing_$playScalaVersion:$akkaVersion",
116+
'akkaProtobuf': "com.typesafe.akka:akka-protobuf-v3_$playScalaVersion:$akkaVersion",
109117
'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3',
110118
'antlr4': 'org.antlr:antlr4:4.9.3',
111119
'assertJ': 'org.assertj:assertj-core:3.11.1',
@@ -350,6 +358,7 @@ allprojects {
350358
}
351359
}
352360
}
361+
353362
}
354363

355364
configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
@@ -393,7 +402,7 @@ subprojects {
393402
implementation externalDependency.annotationApi
394403
constraints {
395404
implementation("com.google.googlejavaformat:google-java-format:$googleJavaFormatVersion")
396-
implementation('io.netty:netty-all:4.1.114.Final')
405+
implementation('io.netty:netty-all:4.1.115.Final')
397406
implementation('org.apache.commons:commons-compress:1.27.1')
398407
implementation('org.apache.velocity:velocity-engine-core:2.4')
399408
implementation('org.hibernate:hibernate-validator:6.0.20.Final')

‎datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -130,8 +130,6 @@ public Object perform(
130130
CallContext ctx = ctxResult.getFirst();
131131
Result result = (Result) ctxResult.getSecond();
132132

133-
setContextRedirectUrl(ctx);
134-
135133
// Handle OIDC authentication errors.
136134
if (OidcResponseErrorHandler.isError(ctx)) {
137135
return OidcResponseErrorHandler.handleError(ctx);
@@ -192,6 +190,9 @@ private Pair<CallContext, Object> superPerform(
192190
}
193191
}
194192

193+
// Set the redirect url from cookie before creating action
194+
setContextRedirectUrl(ctx);
195+
195196
action = this.redirectToOriginallyRequestedUrl(ctx, defaultUrl);
196197
}
197198
} catch (RuntimeException var20) {

‎datahub-frontend/play.gradle

+7
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,13 @@ dependencies {
5555
implementation externalDependency.antlr4Runtime
5656
implementation externalDependency.antlr4
5757
implementation externalDependency.akkaHttp
58+
implementation externalDependency.akkaActor
59+
implementation externalDependency.akkaStream
60+
implementation externalDependency.akkaActorTyped
61+
implementation externalDependency.akkaSlf4j
62+
implementation externalDependency.akkaJackson
63+
implementation externalDependency.akkaParsing
64+
implementation externalDependency.akkaProtobuf
5865

5966
implementation externalDependency.jerseyCore
6067
implementation externalDependency.jerseyGuava

‎datahub-graphql-core/build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ plugins {
33
id "io.github.kobylynskyi.graphql.codegen" version "4.1.1"
44
}
55

6+
apply from: '../gradle/coverage/java-coverage.gradle'
67

78
dependencies {
89
implementation project(':metadata-service:restli-client-api')

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

+19-6
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363
import com.linkedin.datahub.graphql.generated.Domain;
6464
import com.linkedin.datahub.graphql.generated.ERModelRelationship;
6565
import com.linkedin.datahub.graphql.generated.ERModelRelationshipProperties;
66+
import com.linkedin.datahub.graphql.generated.Entity;
6667
import com.linkedin.datahub.graphql.generated.EntityPath;
6768
import com.linkedin.datahub.graphql.generated.EntityRelationship;
6869
import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy;
@@ -312,6 +313,7 @@
312313
import com.linkedin.datahub.graphql.resolvers.type.HyperParameterValueTypeResolver;
313314
import com.linkedin.datahub.graphql.resolvers.type.PlatformSchemaUnionTypeResolver;
314315
import com.linkedin.datahub.graphql.resolvers.type.PropertyValueResolver;
316+
import com.linkedin.datahub.graphql.resolvers.type.ResolvedActorResolver;
315317
import com.linkedin.datahub.graphql.resolvers.type.ResultsTypeResolver;
316318
import com.linkedin.datahub.graphql.resolvers.type.TimeSeriesAspectInterfaceTypeResolver;
317319
import com.linkedin.datahub.graphql.resolvers.user.CreateNativeUserResetTokenResolver;
@@ -1730,12 +1732,22 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) {
17301732
.type(
17311733
"InstitutionalMemoryMetadata",
17321734
typeWiring ->
1733-
typeWiring.dataFetcher(
1734-
"author",
1735-
new LoadableTypeResolver<>(
1736-
corpUserType,
1737-
(env) ->
1738-
((InstitutionalMemoryMetadata) env.getSource()).getAuthor().getUrn())))
1735+
typeWiring
1736+
.dataFetcher(
1737+
"author",
1738+
new LoadableTypeResolver<>(
1739+
corpUserType,
1740+
(env) ->
1741+
((InstitutionalMemoryMetadata) env.getSource())
1742+
.getAuthor()
1743+
.getUrn()))
1744+
.dataFetcher(
1745+
"actor",
1746+
new EntityTypeResolver(
1747+
this.entityTypes,
1748+
(env) ->
1749+
(Entity)
1750+
((InstitutionalMemoryMetadata) env.getSource()).getActor())))
17391751
.type(
17401752
"DatasetStatsSummary",
17411753
typeWiring ->
@@ -2242,6 +2254,7 @@ private void configureTypeResolvers(final RuntimeWiring.Builder builder) {
22422254
"HyperParameterValueType",
22432255
typeWiring -> typeWiring.typeResolver(new HyperParameterValueTypeResolver()))
22442256
.type("PropertyValue", typeWiring -> typeWiring.typeResolver(new PropertyValueResolver()))
2257+
.type("ResolvedActor", typeWiring -> typeWiring.typeResolver(new ResolvedActorResolver()))
22452258
.type("Aspect", typeWiring -> typeWiring.typeResolver(new AspectInterfaceTypeResolver()))
22462259
.type(
22472260
"TimeSeriesAspect",

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java

+16-2
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,21 @@ private TimeSeriesChart getActiveUsersTimeSeriesChart(
8484
final DateTime end,
8585
final String title,
8686
final DateInterval interval) {
87-
final DateRange dateRange =
88-
new DateRange(String.valueOf(beginning.getMillis()), String.valueOf(end.getMillis()));
87+
88+
final DateRange dateRange;
89+
90+
// adjust month to show 1st of month rather than last day of previous month
91+
if (interval == DateInterval.MONTH) {
92+
dateRange =
93+
new DateRange(
94+
String.valueOf(beginning.plusDays(1).getMillis()), // Shift start by 1 day
95+
String.valueOf(end.plusDays(1).getMillis()) // Shift end by 1 day
96+
);
97+
} else {
98+
// week display starting Sundays
99+
dateRange =
100+
new DateRange(String.valueOf(beginning.getMillis()), String.valueOf(end.getMillis()));
101+
}
89102

90103
final List<NamedLine> timeSeriesLines =
91104
_analyticsService.getTimeseriesChart(
@@ -96,6 +109,7 @@ private TimeSeriesChart getActiveUsersTimeSeriesChart(
96109
ImmutableMap.of(),
97110
Collections.emptyMap(),
98111
Optional.of("browserId"));
112+
99113
return TimeSeriesChart.builder()
100114
.setTitle(title)
101115
.setDateRange(dateRange)

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java

+61-21
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,14 @@
11
package com.linkedin.datahub.graphql.analytics.service;
22

3+
import static com.linkedin.metadata.Constants.CORP_USER_EDITABLE_INFO_ASPECT_NAME;
4+
import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME;
35
import static com.linkedin.metadata.Constants.CORP_USER_INFO_ASPECT_NAME;
46

57
import com.google.common.collect.ImmutableSet;
68
import com.linkedin.common.urn.Urn;
79
import com.linkedin.common.urn.UrnUtils;
810
import com.linkedin.dashboard.DashboardInfo;
9-
import com.linkedin.datahub.graphql.generated.BarSegment;
10-
import com.linkedin.datahub.graphql.generated.Cell;
11-
import com.linkedin.datahub.graphql.generated.Entity;
12-
import com.linkedin.datahub.graphql.generated.EntityProfileParams;
13-
import com.linkedin.datahub.graphql.generated.LinkParams;
14-
import com.linkedin.datahub.graphql.generated.NamedBar;
15-
import com.linkedin.datahub.graphql.generated.Row;
16-
import com.linkedin.datahub.graphql.generated.SearchParams;
11+
import com.linkedin.datahub.graphql.generated.*;
1712
import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
1813
import com.linkedin.dataplatform.DataPlatformInfo;
1914
import com.linkedin.dataset.DatasetProperties;
@@ -22,6 +17,7 @@
2217
import com.linkedin.entity.EnvelopedAspect;
2318
import com.linkedin.entity.client.EntityClient;
2419
import com.linkedin.glossary.GlossaryTermInfo;
20+
import com.linkedin.identity.CorpUserEditableInfo;
2521
import com.linkedin.identity.CorpUserInfo;
2622
import com.linkedin.metadata.Constants;
2723
import com.linkedin.metadata.key.GlossaryTermKey;
@@ -35,6 +31,7 @@
3531
import java.util.Set;
3632
import java.util.function.Function;
3733
import java.util.stream.Collectors;
34+
import java.util.stream.Stream;
3835
import javax.annotation.Nonnull;
3936
import javax.annotation.Nullable;
4037
import lombok.extern.slf4j.Slf4j;
@@ -169,36 +166,79 @@ public static void convertToUserInfoRows(
169166
final Map<Urn, EntityResponse> gmsResponseByUser =
170167
entityClient.batchGetV2(
171168
opContext,
172-
CORP_USER_INFO_ASPECT_NAME,
169+
CORP_USER_ENTITY_NAME,
173170
userUrns,
174-
ImmutableSet.of(CORP_USER_INFO_ASPECT_NAME));
175-
final Map<Urn, CorpUserInfo> urnToCorpUserInfo =
171+
ImmutableSet.of(CORP_USER_INFO_ASPECT_NAME, CORP_USER_EDITABLE_INFO_ASPECT_NAME));
172+
final Stream<Map.Entry<Urn, EntityResponse>> entityStream =
176173
gmsResponseByUser.entrySet().stream()
177174
.filter(
178175
entry ->
179176
entry.getValue() != null
180-
&& entry.getValue().getAspects().containsKey(CORP_USER_INFO_ASPECT_NAME))
181-
.collect(
182-
Collectors.toMap(
183-
Map.Entry::getKey,
184-
entry ->
177+
&& (entry.getValue().getAspects().containsKey(CORP_USER_INFO_ASPECT_NAME)
178+
|| entry
179+
.getValue()
180+
.getAspects()
181+
.containsKey(CORP_USER_EDITABLE_INFO_ASPECT_NAME)));
182+
final Map<Urn, Pair<CorpUserInfo, CorpUserEditableInfo>> urnToCorpUserInfo =
183+
entityStream.collect(
184+
Collectors.toMap(
185+
Map.Entry::getKey,
186+
entry -> {
187+
CorpUserInfo userInfo = null;
188+
CorpUserEditableInfo editableInfo = null;
189+
try {
190+
userInfo =
185191
new CorpUserInfo(
186192
entry
187193
.getValue()
188194
.getAspects()
189195
.get(CORP_USER_INFO_ASPECT_NAME)
190196
.getValue()
191-
.data())));
197+
.data());
198+
} catch (Exception e) {
199+
// nothing to do
200+
}
201+
try {
202+
203+
editableInfo =
204+
new CorpUserEditableInfo(
205+
entry
206+
.getValue()
207+
.getAspects()
208+
.get(CORP_USER_EDITABLE_INFO_ASPECT_NAME)
209+
.getValue()
210+
.data());
211+
} catch (Exception e) {
212+
// nothing to do
213+
}
214+
215+
return Pair.of(userInfo, editableInfo);
216+
}));
192217
// Populate a row with the user link, title, and email.
193218
rows.forEach(
194219
row -> {
195220
Urn urn = UrnUtils.getUrn(row.getCells().get(0).getValue());
196221
EntityResponse response = gmsResponseByUser.get(urn);
197222
String maybeDisplayName = response != null ? getUserName(response).orElse(null) : null;
198-
String maybeEmail =
199-
urnToCorpUserInfo.containsKey(urn) ? urnToCorpUserInfo.get(urn).getEmail() : null;
200-
String maybeTitle =
201-
urnToCorpUserInfo.containsKey(urn) ? urnToCorpUserInfo.get(urn).getTitle() : null;
223+
String maybeEmail = null;
224+
String maybeTitle = null;
225+
if (urnToCorpUserInfo.containsKey(urn)) {
226+
Pair<CorpUserInfo, CorpUserEditableInfo> pair = urnToCorpUserInfo.get(urn);
227+
if (pair.getLeft() != null) {
228+
CorpUserInfo userInfo = pair.getLeft();
229+
maybeEmail = userInfo.getEmail();
230+
maybeTitle = userInfo.getTitle();
231+
}
232+
if (pair.getRight() != null) {
233+
CorpUserEditableInfo userInfo = pair.getRight();
234+
if (maybeEmail == null) {
235+
maybeEmail = userInfo.getEmail();
236+
}
237+
if (maybeTitle == null) {
238+
maybeTitle = userInfo.getTitle();
239+
}
240+
}
241+
}
202242
if (maybeDisplayName != null) {
203243
row.getCells().get(0).setValue(maybeDisplayName);
204244
}

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/ResolvedActorResolver.java

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package com.linkedin.datahub.graphql.resolvers.type;
2+
3+
import com.linkedin.datahub.graphql.generated.CorpGroup;
4+
import com.linkedin.datahub.graphql.generated.CorpUser;
5+
import graphql.TypeResolutionEnvironment;
6+
import graphql.schema.GraphQLObjectType;
7+
import graphql.schema.TypeResolver;
8+
9+
public class ResolvedActorResolver implements TypeResolver {
10+
11+
public static final String CORP_USER = "CorpUser";
12+
public static final String CORP_GROUP = "CorpGroup";
13+
14+
@Override
15+
public GraphQLObjectType getType(TypeResolutionEnvironment env) {
16+
if (env.getObject() instanceof CorpUser) {
17+
return env.getSchema().getObjectType(CORP_USER);
18+
} else if (env.getObject() instanceof CorpGroup) {
19+
return env.getSchema().getObjectType(CORP_GROUP);
20+
} else {
21+
throw new RuntimeException(
22+
"Unrecognized object type provided to type resolver, Type:" + env.getObject().toString());
23+
}
24+
}
25+
}

0 commit comments

Comments
 (0)