Skip to content

Commit c7f3a72

Browse files
samanthafigueredo5samanthafigueredo5
authored
Merge branch 'master' into sfiguere_uhg-add-user-location-to-profile
2 parents c1aa21d + 4811de1 commit c7f3a72

File tree

703 files changed

+161804
-8136
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

703 files changed

+161804
-8136
lines changed

‎.github/workflows/build-and-test.yml

+4
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@ jobs:
7575
path: |
7676
~/.cache/uv
7777
key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
78+
- name: Install dependencies
79+
run: ./metadata-ingestion/scripts/install_deps.sh
7880
- name: Set up JDK 17
7981
uses: actions/setup-java@v4
8082
with:
@@ -83,6 +85,7 @@ jobs:
8385
- uses: gradle/actions/setup-gradle@v3
8486
- name: Gradle build (and test) for NOT metadata ingestion
8587
if: ${{ matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
88+
# datahub-schematron:cli excluded due to dependency on metadata-ingestion
8689
run: |
8790
./gradlew build \
8891
-x :metadata-ingestion:build \
@@ -100,6 +103,7 @@ jobs:
100103
-x :metadata-ingestion-modules:gx-plugin:check \
101104
-x :datahub-frontend:build \
102105
-x :datahub-web-react:build \
106+
-x :metadata-integration:java:datahub-schematron:cli:test \
103107
--parallel
104108
- name: Gradle build (and test) for frontend
105109
if: ${{ matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }}

‎.github/workflows/check-datahub-jars.yml

+1
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,5 @@ jobs:
4040
- name: check ${{ matrix.command }} jar
4141
run: |
4242
./gradlew :metadata-integration:java:${{ matrix.command }}:build --info
43+
./gradlew :metadata-integration:java:${{ matrix.command }}:checkShadowJar
4344
./gradlew :metadata-integration:java:${{ matrix.command }}:javadoc

‎.github/workflows/docker-unified.yml

+65
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,11 @@ jobs:
144144
needs: setup
145145
if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
146146
steps:
147+
- name: Free up disk space
148+
run: |
149+
sudo apt-get remove 'dotnet-*' azure-cli || true
150+
sudo rm -rf /usr/local/lib/android/ || true
151+
sudo docker image prune -a -f || true
147152
- name: Set up JDK 17
148153
uses: actions/setup-java@v4
149154
with:
@@ -210,6 +215,11 @@ jobs:
210215
needs: setup
211216
if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
212217
steps:
218+
- name: Free up disk space
219+
run: |
220+
sudo apt-get remove 'dotnet-*' azure-cli || true
221+
sudo rm -rf /usr/local/lib/android/ || true
222+
sudo docker image prune -a -f || true
213223
- name: Set up JDK 17
214224
uses: actions/setup-java@v4
215225
with:
@@ -276,6 +286,11 @@ jobs:
276286
needs: setup
277287
if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
278288
steps:
289+
- name: Free up disk space
290+
run: |
291+
sudo apt-get remove 'dotnet-*' azure-cli || true
292+
sudo rm -rf /usr/local/lib/android/ || true
293+
sudo docker image prune -a -f || true
279294
- name: Set up JDK 17
280295
uses: actions/setup-java@v4
281296
with:
@@ -342,6 +357,11 @@ jobs:
342357
needs: setup
343358
if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
344359
steps:
360+
- name: Free up disk space
361+
run: |
362+
sudo apt-get remove 'dotnet-*' azure-cli || true
363+
sudo rm -rf /usr/local/lib/android/ || true
364+
sudo docker image prune -a -f || true
345365
- name: Set up JDK 17
346366
uses: actions/setup-java@v4
347367
with:
@@ -408,6 +428,11 @@ jobs:
408428
needs: setup
409429
if: ${{ needs.setup.outputs.frontend_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true'}}
410430
steps:
431+
- name: Free up disk space
432+
run: |
433+
sudo apt-get remove 'dotnet-*' azure-cli || true
434+
sudo rm -rf /usr/local/lib/android/ || true
435+
sudo docker image prune -a -f || true
411436
- name: Set up JDK 17
412437
uses: actions/setup-java@v4
413438
with:
@@ -476,6 +501,11 @@ jobs:
476501
needs: setup
477502
if: ${{ needs.setup.outputs.kafka_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true') }}
478503
steps:
504+
- name: Free up disk space
505+
run: |
506+
sudo apt-get remove 'dotnet-*' azure-cli || true
507+
sudo rm -rf /usr/local/lib/android/ || true
508+
sudo docker image prune -a -f || true
479509
- name: Check out the repo
480510
uses: acryldata/sane-checkout-action@v3
481511
- name: Build and push
@@ -532,6 +562,11 @@ jobs:
532562
needs: setup
533563
if: ${{ needs.setup.outputs.mysql_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true') }}
534564
steps:
565+
- name: Free up disk space
566+
run: |
567+
sudo apt-get remove 'dotnet-*' azure-cli || true
568+
sudo rm -rf /usr/local/lib/android/ || true
569+
sudo docker image prune -a -f || true
535570
- name: Check out the repo
536571
uses: acryldata/sane-checkout-action@v3
537572
- name: Build and push
@@ -588,6 +623,11 @@ jobs:
588623
needs: setup
589624
if: ${{ needs.setup.outputs.elasticsearch_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' ) }}
590625
steps:
626+
- name: Free up disk space
627+
run: |
628+
sudo apt-get remove 'dotnet-*' azure-cli || true
629+
sudo rm -rf /usr/local/lib/android/ || true
630+
sudo docker image prune -a -f || true
591631
- name: Check out the repo
592632
uses: acryldata/sane-checkout-action@v3
593633
- name: Build and push
@@ -646,6 +686,11 @@ jobs:
646686
needs: setup
647687
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
648688
steps:
689+
- name: Free up disk space
690+
run: |
691+
sudo apt-get remove 'dotnet-*' azure-cli || true
692+
sudo rm -rf /usr/local/lib/android/ || true
693+
sudo docker image prune -a -f || true
649694
- name: Check out the repo
650695
uses: acryldata/sane-checkout-action@v3
651696
- name: Build and push Base Image
@@ -674,6 +719,11 @@ jobs:
674719
needs: [setup, datahub_ingestion_base_build]
675720
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
676721
steps:
722+
- name: Free up disk space
723+
run: |
724+
sudo apt-get remove 'dotnet-*' azure-cli || true
725+
sudo rm -rf /usr/local/lib/android/ || true
726+
sudo docker image prune -a -f || true
677727
- name: Check out the repo
678728
uses: acryldata/sane-checkout-action@v3
679729
- name: Download Base Image
@@ -717,6 +767,11 @@ jobs:
717767
needs: [setup, datahub_ingestion_base_build]
718768
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
719769
steps:
770+
- name: Free up disk space
771+
run: |
772+
sudo apt-get remove 'dotnet-*' azure-cli || true
773+
sudo rm -rf /usr/local/lib/android/ || true
774+
sudo docker image prune -a -f || true
720775
- name: Check out the repo
721776
uses: acryldata/sane-checkout-action@v3
722777
- name: Download Base Image
@@ -760,6 +815,11 @@ jobs:
760815
needs: [setup, datahub_ingestion_base_slim_build]
761816
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
762817
steps:
818+
- name: Free up disk space
819+
run: |
820+
sudo apt-get remove 'dotnet-*' azure-cli || true
821+
sudo rm -rf /usr/local/lib/android/ || true
822+
sudo docker image prune -a -f || true
763823
- name: Check out the repo
764824
uses: acryldata/sane-checkout-action@v3
765825
- uses: actions/setup-python@v5
@@ -856,6 +916,11 @@ jobs:
856916
needs: [setup, datahub_ingestion_base_full_build]
857917
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
858918
steps:
919+
- name: Free up disk space
920+
run: |
921+
sudo apt-get remove 'dotnet-*' azure-cli || true
922+
sudo rm -rf /usr/local/lib/android/ || true
923+
sudo docker image prune -a -f || true
859924
- name: Check out the repo
860925
uses: acryldata/sane-checkout-action@v3
861926
- uses: actions/setup-python@v5

‎.github/workflows/gx-plugin.yml

+2
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ jobs:
3939
extraPythonRequirement: "great-expectations~=0.16.0 numpy~=1.26.0"
4040
- python-version: "3.11"
4141
extraPythonRequirement: "great-expectations~=0.17.0"
42+
- python-version: "3.11"
43+
extraPythonRequirement: "great-expectations~=0.18.0"
4244
fail-fast: false
4345
steps:
4446
- name: Set up JDK 17

‎.github/workflows/pr-labeler.yml

+5-4
Original file line numberDiff line numberDiff line change
@@ -29,23 +29,24 @@ jobs:
2929
"swaroopjagadish",
3030
"treff7es",
3131
"yoonhyejin",
32-
"eboneil",
3332
"gabe-lyons",
3433
"hsheth2",
3534
"jjoyce0510",
3635
"maggiehays",
3736
"pedro93",
3837
"RyanHolstien",
3938
"sakethvarma397",
40-
"Kunal-kankriya",
4139
"purnimagarg1",
42-
"dushayntAW",
4340
"sagar-salvi-apptware",
4441
"kushagra-apptware",
4542
"Salman-Apptware",
4643
"mayurinehate",
4744
"noggi",
48-
"skrydal"
45+
"skrydal",
46+
"kevinkarchacryl",
47+
"sgomezvillamor",
48+
"acrylJonny",
49+
"chakru-r"
4950
]'),
5051
github.actor
5152
)

‎build.gradle

+12-3
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ buildscript {
4848
// see also datahub-frontend/play.gradle
4949
ext.playVersion = '2.8.22'
5050
ext.playScalaVersion = '2.13'
51+
ext.akkaVersion = '2.6.21' // 2.7.0+ has incompatible license
5152
ext.log4jVersion = '2.23.1'
5253
ext.slf4jVersion = '1.7.36'
5354
ext.logbackClassic = '1.4.14'
@@ -56,7 +57,7 @@ buildscript {
5657
ext.hazelcastVersion = '5.3.6'
5758
ext.ebeanVersion = '15.5.2'
5859
ext.googleJavaFormatVersion = '1.18.1'
59-
ext.openLineageVersion = '1.19.0'
60+
ext.openLineageVersion = '1.24.2'
6061
ext.logbackClassicJava8 = '1.2.12'
6162

6263
ext.docker_registry = 'acryldata'
@@ -105,7 +106,14 @@ project.ext.spec = [
105106
]
106107

107108
project.ext.externalDependency = [
108-
'akkaHttp': "com.typesafe.akka:akka-http-core_$playScalaVersion:10.2.10",
109+
'akkaHttp': "com.typesafe.akka:akka-http-core_$playScalaVersion:10.2.10", // max version due to licensing
110+
'akkaActor': "com.typesafe.akka:akka-actor_$playScalaVersion:$akkaVersion",
111+
'akkaStream': "com.typesafe.akka:akka-stream_$playScalaVersion:$akkaVersion",
112+
'akkaActorTyped': "com.typesafe.akka:akka-actor-typed_$playScalaVersion:$akkaVersion",
113+
'akkaSlf4j': "com.typesafe.akka:akka-slf4j_$playScalaVersion:$akkaVersion",
114+
'akkaJackson': "com.typesafe.akka:akka-serialization-jackson_$playScalaVersion:$akkaVersion",
115+
'akkaParsing': "com.typesafe.akka:akka-parsing_$playScalaVersion:$akkaVersion",
116+
'akkaProtobuf': "com.typesafe.akka:akka-protobuf-v3_$playScalaVersion:$akkaVersion",
109117
'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3',
110118
'antlr4': 'org.antlr:antlr4:4.9.3',
111119
'assertJ': 'org.assertj:assertj-core:3.11.1',
@@ -350,6 +358,7 @@ allprojects {
350358
}
351359
}
352360
}
361+
353362
}
354363

355364
configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
@@ -393,7 +402,7 @@ subprojects {
393402
implementation externalDependency.annotationApi
394403
constraints {
395404
implementation("com.google.googlejavaformat:google-java-format:$googleJavaFormatVersion")
396-
implementation('io.netty:netty-all:4.1.114.Final')
405+
implementation('io.netty:netty-all:4.1.115.Final')
397406
implementation('org.apache.commons:commons-compress:1.27.1')
398407
implementation('org.apache.velocity:velocity-engine-core:2.4')
399408
implementation('org.hibernate:hibernate-validator:6.0.20.Final')

‎datahub-frontend/app/auth/AuthModule.java

+8-5
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import com.google.inject.Provides;
1414
import com.google.inject.Singleton;
1515
import com.google.inject.name.Named;
16+
import com.linkedin.entity.client.EntityClientConfig;
1617
import com.linkedin.entity.client.SystemEntityClient;
1718
import com.linkedin.entity.client.SystemRestliEntityClient;
1819
import com.linkedin.metadata.models.registry.EmptyEntityRegistry;
@@ -213,11 +214,13 @@ protected SystemEntityClient provideEntityClient(
213214

214215
return new SystemRestliEntityClient(
215216
buildRestliClient(),
216-
new ExponentialBackoff(configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)),
217-
configs.getInt(ENTITY_CLIENT_NUM_RETRIES),
218-
configurationProvider.getCache().getClient().getEntityClient(),
219-
Math.max(1, configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE)),
220-
Math.max(1, configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY)));
217+
EntityClientConfig.builder()
218+
.backoffPolicy(new ExponentialBackoff(configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)))
219+
.retryCount(configs.getInt(ENTITY_CLIENT_NUM_RETRIES))
220+
.batchGetV2Size(configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE))
221+
.batchGetV2Concurrency(2)
222+
.build(),
223+
configurationProvider.getCache().getClient().getEntityClient());
221224
}
222225

223226
@Provides

‎datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -130,8 +130,6 @@ public Object perform(
130130
CallContext ctx = ctxResult.getFirst();
131131
Result result = (Result) ctxResult.getSecond();
132132

133-
setContextRedirectUrl(ctx);
134-
135133
// Handle OIDC authentication errors.
136134
if (OidcResponseErrorHandler.isError(ctx)) {
137135
return OidcResponseErrorHandler.handleError(ctx);
@@ -192,6 +190,9 @@ private Pair<CallContext, Object> superPerform(
192190
}
193191
}
194192

193+
// Set the redirect url from cookie before creating action
194+
setContextRedirectUrl(ctx);
195+
195196
action = this.redirectToOriginallyRequestedUrl(ctx, defaultUrl);
196197
}
197198
} catch (RuntimeException var20) {

‎datahub-frontend/play.gradle

+7
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,13 @@ dependencies {
5555
implementation externalDependency.antlr4Runtime
5656
implementation externalDependency.antlr4
5757
implementation externalDependency.akkaHttp
58+
implementation externalDependency.akkaActor
59+
implementation externalDependency.akkaStream
60+
implementation externalDependency.akkaActorTyped
61+
implementation externalDependency.akkaSlf4j
62+
implementation externalDependency.akkaJackson
63+
implementation externalDependency.akkaParsing
64+
implementation externalDependency.akkaProtobuf
5865

5966
implementation externalDependency.jerseyCore
6067
implementation externalDependency.jerseyGuava

‎datahub-graphql-core/build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ plugins {
33
id "io.github.kobylynskyi.graphql.codegen" version "4.1.1"
44
}
55

6+
apply from: '../gradle/coverage/java-coverage.gradle'
67

78
dependencies {
89
implementation project(':metadata-service:restli-client-api')

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

+19-6
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363
import com.linkedin.datahub.graphql.generated.Domain;
6464
import com.linkedin.datahub.graphql.generated.ERModelRelationship;
6565
import com.linkedin.datahub.graphql.generated.ERModelRelationshipProperties;
66+
import com.linkedin.datahub.graphql.generated.Entity;
6667
import com.linkedin.datahub.graphql.generated.EntityPath;
6768
import com.linkedin.datahub.graphql.generated.EntityRelationship;
6869
import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy;
@@ -312,6 +313,7 @@
312313
import com.linkedin.datahub.graphql.resolvers.type.HyperParameterValueTypeResolver;
313314
import com.linkedin.datahub.graphql.resolvers.type.PlatformSchemaUnionTypeResolver;
314315
import com.linkedin.datahub.graphql.resolvers.type.PropertyValueResolver;
316+
import com.linkedin.datahub.graphql.resolvers.type.ResolvedActorResolver;
315317
import com.linkedin.datahub.graphql.resolvers.type.ResultsTypeResolver;
316318
import com.linkedin.datahub.graphql.resolvers.type.TimeSeriesAspectInterfaceTypeResolver;
317319
import com.linkedin.datahub.graphql.resolvers.user.CreateNativeUserResetTokenResolver;
@@ -1730,12 +1732,22 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) {
17301732
.type(
17311733
"InstitutionalMemoryMetadata",
17321734
typeWiring ->
1733-
typeWiring.dataFetcher(
1734-
"author",
1735-
new LoadableTypeResolver<>(
1736-
corpUserType,
1737-
(env) ->
1738-
((InstitutionalMemoryMetadata) env.getSource()).getAuthor().getUrn())))
1735+
typeWiring
1736+
.dataFetcher(
1737+
"author",
1738+
new LoadableTypeResolver<>(
1739+
corpUserType,
1740+
(env) ->
1741+
((InstitutionalMemoryMetadata) env.getSource())
1742+
.getAuthor()
1743+
.getUrn()))
1744+
.dataFetcher(
1745+
"actor",
1746+
new EntityTypeResolver(
1747+
this.entityTypes,
1748+
(env) ->
1749+
(Entity)
1750+
((InstitutionalMemoryMetadata) env.getSource()).getActor())))
17391751
.type(
17401752
"DatasetStatsSummary",
17411753
typeWiring ->
@@ -2242,6 +2254,7 @@ private void configureTypeResolvers(final RuntimeWiring.Builder builder) {
22422254
"HyperParameterValueType",
22432255
typeWiring -> typeWiring.typeResolver(new HyperParameterValueTypeResolver()))
22442256
.type("PropertyValue", typeWiring -> typeWiring.typeResolver(new PropertyValueResolver()))
2257+
.type("ResolvedActor", typeWiring -> typeWiring.typeResolver(new ResolvedActorResolver()))
22452258
.type("Aspect", typeWiring -> typeWiring.typeResolver(new AspectInterfaceTypeResolver()))
22462259
.type(
22472260
"TimeSeriesAspect",

0 commit comments

Comments
 (0)