Skip to content

Commit aab5d0a

Browse files
rtekalrtekal
authored
Merge pull request #16 from rtekal/rtekal-pl-join-ui-ERModelRelation
Rtekal pl join UI er model relation
2 parents dd09115 + 2831bf6 commit aab5d0a

File tree

486 files changed

+30290
-8664
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

486 files changed

+30290
-8664
lines changed

‎.github/workflows/airflow-plugin.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,8 @@ jobs:
8787
token: ${{ secrets.CODECOV_TOKEN }}
8888
directory: .
8989
fail_ci_if_error: false
90-
flags: airflow-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
91-
name: pytest-airflow
90+
flags: airflow,airflow-${{ matrix.extra_pip_extras }}
91+
name: pytest-airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_requirements }}
9292
verbose: true
9393

9494
event-file:

‎.github/workflows/build-and-test.yml

+4
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,10 @@ jobs:
8383
./gradlew :datahub-frontend:build :datahub-web-react:build --parallel
8484
env:
8585
NODE_OPTIONS: "--max-old-space-size=3072"
86+
- name: Gradle compile (jdk8) for legacy Spark
87+
if: ${{ matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
88+
run: |
89+
./gradlew -PjavaClassVersionDefault=8 :metadata-integration:java:spark-lineage:compileJava
8690
- uses: actions/upload-artifact@v3
8791
if: always()
8892
with:

‎.github/workflows/docker-unified.yml

+12
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,18 @@ jobs:
6464
steps:
6565
- name: Check out the repo
6666
uses: hsheth2/sane-checkout-action@v1
67+
- uses: actions/setup-python@v4
68+
with:
69+
python-version: "3.10"
70+
cache: "pip"
71+
- name: Set up JDK 17
72+
uses: actions/setup-java@v3
73+
with:
74+
distribution: "zulu"
75+
java-version: 17
76+
- name: Run lint on smoke test
77+
run: |
78+
./gradlew :smoke-test:lint
6779
- name: Compute Tag
6880
id: tag
6981
run: |

‎.github/workflows/metadata-ingestion.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ jobs:
3131
# DATAHUB_LOOKML_GIT_TEST_SSH_KEY: ${{ secrets.DATAHUB_LOOKML_GIT_TEST_SSH_KEY }}
3232
strategy:
3333
matrix:
34-
python-version: ["3.7", "3.10"]
34+
python-version: ["3.8", "3.10"]
3535
command:
3636
[
3737
"testQuick",
@@ -40,7 +40,7 @@ jobs:
4040
"testIntegrationBatch2",
4141
]
4242
include:
43-
- python-version: "3.7"
43+
- python-version: "3.8"
4444
- python-version: "3.10"
4545
fail-fast: false
4646
steps:

‎.github/workflows/spark-smoke-test.yml

+17-1
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,12 @@ jobs:
4242
cache: "pip"
4343
- name: Install dependencies
4444
run: ./metadata-ingestion/scripts/install_deps.sh
45+
- name: Disk Check
46+
run: df -h . && docker images
4547
- name: Remove images
4648
run: docker image prune -a -f || true
49+
- name: Disk Check
50+
run: df -h . && docker images
4751
- name: Smoke test
4852
run: |
4953
./gradlew :metadata-integration:java:spark-lineage:integrationTest \
@@ -54,12 +58,24 @@ jobs:
5458
-x :datahub-web-react:yarnBuild \
5559
-x :datahub-web-react:distZip \
5660
-x :datahub-web-react:jar
61+
- name: store logs
62+
if: failure()
63+
run: |
64+
docker ps -a
65+
docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log || true
66+
docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log || true
67+
docker logs broker >& broker-${{ matrix.test_strategy }}.log || true
68+
docker logs mysql >& mysql-${{ matrix.test_strategy }}.log || true
69+
docker logs elasticsearch >& elasticsearch-${{ matrix.test_strategy }}.log || true
70+
docker logs datahub-frontend-react >& frontend-${{ matrix.test_strategy }}.log || true
5771
- name: Upload logs
5872
uses: actions/upload-artifact@v3
5973
if: failure()
6074
with:
6175
name: docker logs
62-
path: "docker/build/container-logs/*.log"
76+
path: |
77+
"**/build/container-logs/*.log"
78+
"*.log"
6379
- uses: actions/upload-artifact@v3
6480
if: always()
6581
with:

‎build.gradle

+87-18
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,32 @@
11
buildscript {
2-
ext.jdkVersion = 17
3-
ext.javaClassVersion = 11
2+
ext.jdkVersionDefault = 17
3+
ext.javaClassVersionDefault = 11
4+
5+
ext.jdkVersion = { p ->
6+
// If Spring 6 is present, hard dependency on jdk17
7+
if (p.configurations.any { it.getDependencies().any{
8+
(it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6."))
9+
|| (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test"))
10+
}}) {
11+
return 17
12+
} else {
13+
// otherwise we can use the preferred default which can be overridden with a property: -PjdkVersionDefault
14+
return p.hasProperty('jdkVersionDefault') ? Integer.valueOf((String) p.getProperty('jdkVersionDefault')) : ext.jdkVersionDefault
15+
}
16+
}
17+
18+
ext.javaClassVersion = { p ->
19+
// If Spring 6 is present, hard dependency on jdk17
20+
if (p.configurations.any { it.getDependencies().any{
21+
(it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6."))
22+
|| (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test"))
23+
}}) {
24+
return 17
25+
} else {
26+
// otherwise we can use the preferred default which can be overridden with a property: -PjavaClassVersionDefault
27+
return p.hasProperty('javaClassVersionDefault') ? Integer.valueOf((String) p.getProperty('javaClassVersionDefault')) : ext.javaClassVersionDefault
28+
}
29+
}
430

531
ext.junitJupiterVersion = '5.6.1'
632
// Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
@@ -217,6 +243,7 @@ project.ext.externalDependency = [
217243
'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
218244
'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15',
219245
'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46',
246+
'springBootAutoconfigureJdk11': 'org.springframework.boot:spring-boot-autoconfigure:2.7.18',
220247
'testng': 'org.testng:testng:7.8.0',
221248
'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion,
222249
'testContainersJunit': 'org.testcontainers:junit-jupiter:' + testContainersVersion,
@@ -252,48 +279,56 @@ allprojects {
252279
}
253280
}
254281

255-
if (project.plugins.hasPlugin('java')
282+
/**
283+
* If making changes to this section also see the sections for pegasus below
284+
* which use project.plugins.hasPlugin('pegasus')
285+
**/
286+
if (!project.plugins.hasPlugin('pegasus') && (project.plugins.hasPlugin('java')
256287
|| project.plugins.hasPlugin('java-library')
257-
|| project.plugins.hasPlugin('application')
258-
|| project.plugins.hasPlugin('pegasus')) {
288+
|| project.plugins.hasPlugin('application'))) {
259289

260290
java {
261291
toolchain {
262-
languageVersion = JavaLanguageVersion.of(jdkVersion)
292+
languageVersion = JavaLanguageVersion.of(jdkVersion(project))
263293
}
264294
}
265295

266296
compileJava {
267-
options.release = javaClassVersion
297+
options.release = javaClassVersion(project)
268298
}
299+
269300
tasks.withType(JavaCompile).configureEach {
270301
javaCompiler = javaToolchains.compilerFor {
271-
languageVersion = JavaLanguageVersion.of(jdkVersion)
302+
languageVersion = JavaLanguageVersion.of(jdkVersion(project))
272303
}
273304
// Puts parameter names into compiled class files, necessary for Spring 6
274305
options.compilerArgs.add("-parameters")
275306
}
276307

277308
tasks.withType(JavaExec).configureEach {
278309
javaLauncher = javaToolchains.launcherFor {
279-
languageVersion = JavaLanguageVersion.of(jdkVersion)
310+
languageVersion = JavaLanguageVersion.of(jdkVersion(project))
280311
}
281312
}
313+
}
314+
315+
// not duplicated, need to set this outside and inside afterEvaluate
316+
afterEvaluate {
317+
/**
318+
* If making changes to this section also see the sections for pegasus below
319+
* which use project.plugins.hasPlugin('pegasus')
320+
**/
321+
if (!project.plugins.hasPlugin('pegasus') && (project.plugins.hasPlugin('java')
322+
|| project.plugins.hasPlugin('java-library')
323+
|| project.plugins.hasPlugin('application'))) {
282324

283-
// not duplicated, need to set this outside and inside afterEvaluate
284-
afterEvaluate {
285325
compileJava {
286-
options.release = javaClassVersion
287-
}
288-
tasks.withType(JavaCompile).configureEach {
289-
javaCompiler = javaToolchains.compilerFor {
290-
languageVersion = JavaLanguageVersion.of(jdkVersion)
291-
}
326+
options.release = javaClassVersion(project)
292327
}
293328

294329
tasks.withType(JavaExec).configureEach {
295330
javaLauncher = javaToolchains.launcherFor {
296-
languageVersion = JavaLanguageVersion.of(jdkVersion)
331+
languageVersion = JavaLanguageVersion.of(jdkVersion(project))
297332
}
298333
}
299334
}
@@ -368,6 +403,30 @@ subprojects {
368403
dataTemplateCompile externalDependency.annotationApi // support > jdk8
369404
restClientCompile spec.product.pegasus.restliClient
370405
}
406+
407+
java {
408+
toolchain {
409+
languageVersion = JavaLanguageVersion.of(jdkVersion(project))
410+
}
411+
}
412+
413+
compileJava {
414+
options.release = javaClassVersion(project)
415+
}
416+
417+
tasks.withType(JavaCompile).configureEach {
418+
javaCompiler = javaToolchains.compilerFor {
419+
languageVersion = JavaLanguageVersion.of(jdkVersion(project))
420+
}
421+
// Puts parameter names into compiled class files, necessary for Spring 6
422+
options.compilerArgs.add("-parameters")
423+
}
424+
425+
tasks.withType(JavaExec).configureEach {
426+
javaLauncher = javaToolchains.launcherFor {
427+
languageVersion = JavaLanguageVersion.of(jdkVersion(project))
428+
}
429+
}
371430
}
372431

373432
afterEvaluate {
@@ -394,6 +453,16 @@ subprojects {
394453
dataTemplateCompile externalDependency.annotationApi // support > jdk8
395454
restClientCompile spec.product.pegasus.restliClient
396455
}
456+
457+
compileJava {
458+
options.release = javaClassVersion(project)
459+
}
460+
461+
tasks.withType(JavaExec).configureEach {
462+
javaLauncher = javaToolchains.launcherFor {
463+
languageVersion = JavaLanguageVersion.of(jdkVersion(project))
464+
}
465+
}
397466
}
398467
}
399468
}

‎datahub-frontend/app/auth/AuthUtils.java

+3
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,9 @@ public class AuthUtils {
7676
public static final String USE_NONCE = "useNonce";
7777
public static final String READ_TIMEOUT = "readTimeout";
7878
public static final String EXTRACT_JWT_ACCESS_TOKEN_CLAIMS = "extractJwtAccessTokenClaims";
79+
// Retained for backwards compatibility
80+
public static final String PREFERRED_JWS_ALGORITHM = "preferredJwsAlgorithm";
81+
public static final String PREFERRED_JWS_ALGORITHM_2 = "preferredJwsAlgorithm2";
7982

8083
/**
8184
* Determines whether the inbound request should be forward to downstream Metadata Service. Today,

‎datahub-frontend/app/auth/sso/oidc/OidcConfigs.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -226,8 +226,8 @@ public Builder from(final com.typesafe.config.Config configs, final String ssoSe
226226
extractJwtAccessTokenClaims =
227227
Optional.of(jsonNode.get(EXTRACT_JWT_ACCESS_TOKEN_CLAIMS).asBoolean());
228228
}
229-
if (jsonNode.has(OIDC_PREFERRED_JWS_ALGORITHM)) {
230-
preferredJwsAlgorithm = Optional.of(jsonNode.get(OIDC_PREFERRED_JWS_ALGORITHM).asText());
229+
if (jsonNode.has(PREFERRED_JWS_ALGORITHM_2)) {
230+
preferredJwsAlgorithm = Optional.of(jsonNode.get(PREFERRED_JWS_ALGORITHM_2).asText());
231231
} else {
232232
preferredJwsAlgorithm =
233233
Optional.ofNullable(getOptional(configs, OIDC_PREFERRED_JWS_ALGORITHM, null));

‎datahub-frontend/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ docker {
8787
}
8888
}
8989

90-
task unversionZip(type: Copy, dependsOn: [':datahub-web-react:build', dist]) {
90+
task unversionZip(type: Copy, dependsOn: [':datahub-web-react:distZip', dist]) {
9191
from ("${buildDir}/distributions")
9292
include "datahub-frontend-${version}.zip"
9393
into "${buildDir}/docker/"

‎datahub-frontend/play.gradle

+3
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,9 @@ play {
101101
test {
102102
useJUnitPlatform()
103103

104+
testLogging.showStandardStreams = true
105+
testLogging.exceptionFormat = 'full'
106+
104107
def playJava17CompatibleJvmArgs = [
105108
"--add-opens=java.base/java.lang=ALL-UNNAMED",
106109
//"--add-opens=java.base/java.lang.invoke=ALL-UNNAMED",

‎datahub-frontend/test/security/OidcConfigurationTest.java

+24
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package security;
22

3+
import static auth.AuthUtils.*;
34
import static auth.sso.oidc.OidcConfigs.*;
45
import static org.junit.jupiter.api.Assertions.assertEquals;
56

@@ -24,6 +25,7 @@
2425
import java.util.concurrent.TimeUnit;
2526
import org.junit.jupiter.api.Test;
2627
import org.pac4j.oidc.client.OidcClient;
28+
import org.json.JSONObject;
2729

2830
public class OidcConfigurationTest {
2931

@@ -317,4 +319,26 @@ public void readTimeoutPropagation() {
317319
OidcProvider oidcProvider = new OidcProvider(oidcConfigs);
318320
assertEquals(10000, ((OidcClient) oidcProvider.client()).getConfiguration().getReadTimeout());
319321
}
322+
323+
@Test
324+
public void readPreferredJwsAlgorithmPropagationFromConfig() {
325+
final String SSO_SETTINGS_JSON_STR = new JSONObject().put(PREFERRED_JWS_ALGORITHM, "HS256").toString();
326+
CONFIG.withValue(OIDC_PREFERRED_JWS_ALGORITHM, ConfigValueFactory.fromAnyRef("RS256"));
327+
OidcConfigs.Builder oidcConfigsBuilder = new OidcConfigs.Builder();
328+
oidcConfigsBuilder.from(CONFIG, SSO_SETTINGS_JSON_STR);
329+
OidcConfigs oidcConfigs = new OidcConfigs(oidcConfigsBuilder);
330+
OidcProvider oidcProvider = new OidcProvider(oidcConfigs);
331+
assertEquals("RS256", ((OidcClient) oidcProvider.client()).getConfiguration().getPreferredJwsAlgorithm().toString());
332+
}
333+
334+
@Test
335+
public void readPreferredJwsAlgorithmPropagationFromJSON() {
336+
final String SSO_SETTINGS_JSON_STR = new JSONObject().put(PREFERRED_JWS_ALGORITHM, "Unused").put(PREFERRED_JWS_ALGORITHM_2, "HS256").toString();
337+
CONFIG.withValue(OIDC_PREFERRED_JWS_ALGORITHM, ConfigValueFactory.fromAnyRef("RS256"));
338+
OidcConfigs.Builder oidcConfigsBuilder = new OidcConfigs.Builder();
339+
oidcConfigsBuilder.from(CONFIG, SSO_SETTINGS_JSON_STR);
340+
OidcConfigs oidcConfigs = new OidcConfigs(oidcConfigsBuilder);
341+
OidcProvider oidcProvider = new OidcProvider(oidcConfigs);
342+
assertEquals("HS256", ((OidcClient) oidcProvider.client()).getConfiguration().getPreferredJwsAlgorithm().toString());
343+
}
320344
}

‎datahub-graphql-core/build.gradle

+4-18
Original file line numberDiff line numberDiff line change
@@ -31,30 +31,16 @@ dependencies {
3131

3232
graphqlCodegen {
3333
// For options: https://github.com/kobylynskyi/graphql-java-codegen/blob/master/docs/codegen-options.md
34-
graphqlSchemaPaths = [
35-
"$projectDir/src/main/resources/entity.graphql".toString(),
36-
"$projectDir/src/main/resources/app.graphql".toString(),
37-
"$projectDir/src/main/resources/search.graphql".toString(),
38-
"$projectDir/src/main/resources/analytics.graphql".toString(),
39-
"$projectDir/src/main/resources/recommendation.graphql".toString(),
40-
"$projectDir/src/main/resources/ingestion.graphql".toString(),
41-
"$projectDir/src/main/resources/auth.graphql".toString(),
42-
"$projectDir/src/main/resources/timeline.graphql".toString(),
43-
"$projectDir/src/main/resources/tests.graphql".toString(),
44-
"$projectDir/src/main/resources/properties.graphql".toString(),
45-
"$projectDir/src/main/resources/step.graphql".toString(),
46-
"$projectDir/src/main/resources/lineage.graphql".toString(),
47-
"$projectDir/src/main/resources/forms.graphql".toString()
48-
]
49-
outputDir = new File("$projectDir/src/mainGeneratedGraphQL/java")
34+
graphqlSchemaPaths = fileTree(dir: "${projectDir}/src/main/resources", include: '**/*.graphql').collect { it.absolutePath }
35+
outputDir = new File("${projectDir}/src/mainGeneratedGraphQL/java")
5036
packageName = "com.linkedin.datahub.graphql.generated"
5137
generateToString = true
5238
generateApis = true
5339
generateParameterizedFieldsResolvers = false
5440
modelValidationAnnotation = "@javax.annotation.Nonnull"
5541
customTypesMapping = [
56-
Long: "Long",
57-
Float: "Float"
42+
Long: "Long",
43+
Float: "Float"
5844
]
5945
}
6046

0 commit comments

Comments
 (0)