Skip to content

Commit a2991a6

Browse files
jjoyce0510jjoyce0510
authored
Merge branch 'master' into jj--styling-assertions-table-hotfixes-oss
2 parents b500b44 + e59f497 commit a2991a6

File tree

186 files changed

+9160
-1726
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

186 files changed

+9160
-1726
lines changed

‎.github/workflows/pr-labeler.yml

+4-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,10 @@ jobs:
5252
"chakru-r",
5353
"brock-acryl",
5454
"mminichino",
55-
"jayacryl"
55+
"jayacryl",
56+
"v-tarasevich-blitz-brain",
57+
"ryota-cloud",
58+
"annadoesdesign"
5659
]'),
5760
github.actor
5861
)

‎build.gradle

+50-15
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
import org.apache.tools.ant.filters.ReplaceTokens
2+
3+
14
buildscript {
25
ext.jdkVersionDefault = 17
36
ext.javaClassVersionDefault = 11
@@ -60,6 +63,7 @@ buildscript {
6063
ext.googleJavaFormatVersion = '1.18.1'
6164
ext.openLineageVersion = '1.25.0'
6265
ext.logbackClassicJava8 = '1.2.12'
66+
ext.awsSdk2Version = '2.30.33'
6367

6468
ext.docker_registry = 'acryldata'
6569

@@ -120,12 +124,12 @@ project.ext.externalDependency = [
120124
'assertJ': 'org.assertj:assertj-core:3.11.1',
121125
'avro': 'org.apache.avro:avro:1.11.4',
122126
'avroCompiler': 'org.apache.avro:avro-compiler:1.11.4',
123-
'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.17',
124-
'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:2.0.3',
125-
'awsS3': 'software.amazon.awssdk:s3:2.26.21',
126-
'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.13',
127-
'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.2',
128-
'awsRds':'software.amazon.awssdk:rds:2.18.24',
127+
'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.23',
128+
'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:2.3.0',
129+
'awsS3': "software.amazon.awssdk:s3:$awsSdk2Version",
130+
'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.15',
131+
'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:2.5.4',
132+
'awsRds':"software.amazon.awssdk:rds:$awsSdk2Version",
129133
'cacheApi': 'javax.cache:cache-api:1.1.0',
130134
'commonsCli': 'commons-cli:commons-cli:1.5.0',
131135
'commonsIo': 'commons-io:commons-io:2.17.0',
@@ -240,7 +244,7 @@ project.ext.externalDependency = [
240244
'playFilters': "com.typesafe.play:filters-helpers_$playScalaVersion:$playVersion",
241245
'pac4j': 'org.pac4j:pac4j-oidc:6.0.6',
242246
'playPac4j': "org.pac4j:play-pac4j_$playScalaVersion:12.0.0-PLAY2.8",
243-
'postgresql': 'org.postgresql:postgresql:42.7.4',
247+
'postgresql': 'org.postgresql:postgresql:42.7.5',
244248
'protobuf': 'com.google.protobuf:protobuf-java:3.25.5',
245249
'grpcProtobuf': 'io.grpc:grpc-protobuf:1.53.0',
246250
'rangerCommons': 'org.apache.ranger:ranger-plugins-common:2.3.0',
@@ -395,25 +399,56 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
395399
}
396400
}
397401

402+
apply plugin: 'com.gorylenko.gradle-git-properties'
403+
gitProperties {
404+
keys = ['git.commit.id','git.commit.id.describe','git.commit.time']
405+
// using any tags (not limited to annotated tags) for "git.commit.id.describe" property
406+
// see http://ajoberstar.org/grgit/grgit-describe.html for more info about the describe method and available parameters
407+
// 'it' is an instance of org.ajoberstar.grgit.Grgit
408+
customProperty 'git.commit.id.describe', { it.describe(tags: true) }
409+
gitPropertiesResourceDir = rootProject.buildDir
410+
failOnNoGitDirectory = false
411+
}
412+
413+
def gitPropertiesGenerated = false
414+
415+
apply from: 'gradle/versioning/versioning-global.gradle'
416+
417+
tasks.register("generateGitPropertiesGlobal", com.gorylenko.GenerateGitPropertiesTask) {
418+
doFirst {
419+
if (!gitPropertiesGenerated) {
420+
println "Generating git.properties"
421+
gitPropertiesGenerated = true
422+
} else {
423+
// Skip actual execution if already run
424+
onlyIf { false }
425+
}
426+
}
427+
}
428+
398429
subprojects {
399430

400431
apply plugin: 'maven-publish'
401-
apply plugin: 'com.gorylenko.gradle-git-properties'
402432
apply plugin: 'com.diffplug.spotless'
403433

404-
gitProperties {
405-
keys = ['git.commit.id','git.commit.id.describe','git.commit.time']
406-
// using any tags (not limited to annotated tags) for "git.commit.id.describe" property
407-
// see http://ajoberstar.org/grgit/grgit-describe.html for more info about the describe method and available parameters
408-
// 'it' is an instance of org.ajoberstar.grgit.Grgit
409-
customProperty 'git.commit.id.describe', { it.describe(tags: true) }
410-
failOnNoGitDirectory = false
434+
def gitPropertiesTask = tasks.register("copyGitProperties", Copy) {
435+
dependsOn rootProject.tasks.named("generateGitPropertiesGlobal")
436+
def sourceFile = file("${rootProject.buildDir}/git.properties")
437+
from sourceFile
438+
into "$project.buildDir/resources/main"
411439
}
412440

413441
plugins.withType(JavaPlugin).configureEach {
442+
project.tasks.named(JavaPlugin.CLASSES_TASK_NAME).configure{
443+
dependsOn gitPropertiesTask
444+
}
414445
if (project.name == 'datahub-web-react') {
415446
return
416447
}
448+
/* TODO: evaluate ignoring jar timestamps for increased caching (compares checksum instead)
449+
jar {
450+
preserveFileTimestamps = false
451+
}*/
417452

418453
dependencies {
419454
implementation externalDependency.annotationApi

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java

+3
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ public CompletableFuture<AppConfig> get(final DataFetchingEnvironment environmen
140140
visualConfig.setAppTitle(_visualConfiguration.getAppTitle());
141141
}
142142
visualConfig.setHideGlossary(_visualConfiguration.isHideGlossary());
143+
visualConfig.setShowFullTitleInLineage(_visualConfiguration.isShowFullTitleInLineage());
143144
}
144145
if (_visualConfiguration != null && _visualConfiguration.getQueriesTab() != null) {
145146
QueriesTabConfig queriesTabConfig = new QueriesTabConfig();
@@ -204,6 +205,8 @@ public CompletableFuture<AppConfig> get(final DataFetchingEnvironment environmen
204205
.setShowNavBarRedesign(_featureFlags.isShowNavBarRedesign())
205206
.setShowAutoCompleteResults(_featureFlags.isShowAutoCompleteResults())
206207
.setEntityVersioningEnabled(_featureFlags.isEntityVersioning())
208+
.setShowSearchBarAutocompleteRedesign(
209+
_featureFlags.isShowSearchBarAutocompleteRedesign())
207210
.build();
208211

209212
appConfig.setFeatureFlags(featureFlagsConfig);

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/util/SystemMetadataUtils.java

+24-39
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@
22

33
import static com.linkedin.metadata.Constants.DEFAULT_RUN_ID;
44

5+
import com.linkedin.entity.EnvelopedAspect;
56
import com.linkedin.entity.EnvelopedAspectMap;
67
import com.linkedin.mxe.SystemMetadata;
7-
import java.util.ArrayList;
8-
import java.util.List;
8+
import java.util.Comparator;
9+
import java.util.Objects;
10+
import java.util.Optional;
911
import javax.annotation.Nonnull;
1012
import javax.annotation.Nullable;
1113

@@ -15,51 +17,34 @@ private SystemMetadataUtils() {}
1517

1618
@Nullable
1719
public static Long getLastIngestedTime(@Nonnull EnvelopedAspectMap aspectMap) {
18-
RunInfo lastIngestionRun = getLastIngestionRun(aspectMap);
19-
return lastIngestionRun != null ? lastIngestionRun.getTime() : null;
20+
return getLastIngestionRun(aspectMap).map(RunInfo::getTime).orElse(null);
2021
}
2122

2223
@Nullable
2324
public static String getLastIngestedRunId(@Nonnull EnvelopedAspectMap aspectMap) {
24-
RunInfo lastIngestionRun = getLastIngestionRun(aspectMap);
25-
return lastIngestionRun != null ? lastIngestionRun.getId() : null;
25+
return getLastIngestionRun(aspectMap).map(RunInfo::getId).orElse(null);
2626
}
2727

2828
/**
29-
* Returns a sorted list of all of the most recent ingestion runs based on the most recent aspects
30-
* present for the entity.
29+
* Returns the most recent ingestion run based on the most recent aspects present for the entity.
3130
*/
3231
@Nonnull
33-
public static List<RunInfo> getLastIngestionRuns(@Nonnull EnvelopedAspectMap aspectMap) {
34-
final List<RunInfo> runs = new ArrayList<>();
35-
for (String aspect : aspectMap.keySet()) {
36-
if (aspectMap.get(aspect).hasSystemMetadata()) {
37-
SystemMetadata systemMetadata = aspectMap.get(aspect).getSystemMetadata();
38-
if (systemMetadata.hasLastRunId()
39-
&& !systemMetadata.getLastRunId().equals(DEFAULT_RUN_ID)
40-
&& systemMetadata.hasLastObserved()) {
41-
Long lastObserved = systemMetadata.getLastObserved();
42-
String runId = systemMetadata.getLastRunId();
43-
RunInfo run = new RunInfo(runId, lastObserved);
44-
runs.add(run);
45-
} else if (systemMetadata.hasRunId()
46-
&& !systemMetadata.getRunId().equals(DEFAULT_RUN_ID)
47-
&& systemMetadata.hasLastObserved()) {
48-
// Handle the legacy case: Check original run ids.
49-
Long lastObserved = systemMetadata.getLastObserved();
50-
String runId = systemMetadata.getRunId();
51-
RunInfo run = new RunInfo(runId, lastObserved);
52-
runs.add(run);
53-
}
54-
}
55-
}
56-
runs.sort((a, b) -> Long.compare(b.getTime(), a.getTime()));
57-
return runs;
58-
}
59-
60-
@Nullable
61-
private static RunInfo getLastIngestionRun(@Nonnull EnvelopedAspectMap aspectMap) {
62-
List<RunInfo> runs = getLastIngestionRuns(aspectMap);
63-
return !runs.isEmpty() ? runs.get(0) : null; // Just take the first, to get the most recent run.
32+
private static Optional<RunInfo> getLastIngestionRun(@Nonnull EnvelopedAspectMap aspectMap) {
33+
return aspectMap.values().stream()
34+
.filter(EnvelopedAspect::hasSystemMetadata)
35+
.map(EnvelopedAspect::getSystemMetadata)
36+
.filter(SystemMetadata::hasLastObserved)
37+
.map(
38+
systemMetadata ->
39+
Optional.ofNullable(systemMetadata.getLastRunId())
40+
.filter(lastRunId -> !lastRunId.equals(DEFAULT_RUN_ID))
41+
.or(
42+
() ->
43+
Optional.ofNullable(systemMetadata.getRunId())
44+
.filter(runId -> !runId.equals(DEFAULT_RUN_ID)))
45+
.map(runId -> new RunInfo(runId, systemMetadata.getLastObserved()))
46+
.orElse(null))
47+
.filter(Objects::nonNull)
48+
.max(Comparator.comparingLong(RunInfo::getTime));
6449
}
6550
}

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainType.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ public class DomainType
4040
Constants.OWNERSHIP_ASPECT_NAME,
4141
Constants.INSTITUTIONAL_MEMORY_ASPECT_NAME,
4242
Constants.STRUCTURED_PROPERTIES_ASPECT_NAME,
43-
Constants.FORMS_ASPECT_NAME);
43+
Constants.FORMS_ASPECT_NAME,
44+
Constants.DISPLAY_PROPERTIES_ASPECT_NAME);
4445
private final EntityClient _entityClient;
4546

4647
public DomainType(final EntityClient entityClient) {

‎datahub-graphql-core/src/main/resources/app.graphql

+10
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,11 @@ type VisualConfig {
276276
Configuration for search results
277277
"""
278278
searchResult: SearchResultsVisualConfig
279+
280+
"""
281+
Show full title in lineage view by default
282+
"""
283+
showFullTitleInLineage: Boolean
279284
}
280285

281286
"""
@@ -593,6 +598,11 @@ type FeatureFlagsConfig {
593598
If turned on, exposes the versioning feature by allowing users to link entities in the UI.
594599
"""
595600
entityVersioningEnabled: Boolean!
601+
602+
"""
603+
If turned on, show the redesigned search bar's autocomplete
604+
"""
605+
showSearchBarAutocompleteRedesign: Boolean!
596606
}
597607

598608
"""

‎datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/SystemMetadataUtilsTest.java

-28
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,10 @@
33
import static com.linkedin.metadata.Constants.DEFAULT_RUN_ID;
44
import static org.testng.Assert.*;
55

6-
import com.linkedin.datahub.graphql.types.common.mappers.util.RunInfo;
76
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
87
import com.linkedin.entity.EnvelopedAspect;
98
import com.linkedin.entity.EnvelopedAspectMap;
109
import com.linkedin.mxe.SystemMetadata;
11-
import java.util.List;
1210
import org.testng.annotations.Test;
1311

1412
public class SystemMetadataUtilsTest {
@@ -63,32 +61,6 @@ public void testGetLastIngestedRunId() {
6361
assertEquals(lastRunId, "real-id-1");
6462
}
6563

66-
@Test
67-
public void testGetLastIngestedRuns() {
68-
EnvelopedAspectMap aspectMap = new EnvelopedAspectMap();
69-
aspectMap.put(
70-
"default-run-id",
71-
new EnvelopedAspect()
72-
.setSystemMetadata(
73-
new SystemMetadata().setRunId(DEFAULT_RUN_ID).setLastObserved(recentLastObserved)));
74-
aspectMap.put(
75-
"real-run-id",
76-
new EnvelopedAspect()
77-
.setSystemMetadata(
78-
new SystemMetadata().setRunId("real-id-1").setLastObserved(mediumLastObserved)));
79-
aspectMap.put(
80-
"real-run-id2",
81-
new EnvelopedAspect()
82-
.setSystemMetadata(
83-
new SystemMetadata().setRunId("real-id-2").setLastObserved(distantLastObserved)));
84-
85-
List<RunInfo> runs = SystemMetadataUtils.getLastIngestionRuns(aspectMap);
86-
87-
assertEquals(runs.size(), 2);
88-
assertEquals(runs.get(0), new RunInfo("real-id-1", mediumLastObserved));
89-
assertEquals(runs.get(1), new RunInfo("real-id-2", distantLastObserved));
90-
}
91-
9264
@Test
9365
public void testGetLastIngestedTimeAllDefaultRunIds() {
9466
EnvelopedAspectMap aspectMap = new EnvelopedAspectMap();

‎datahub-web-react/build.gradle

+1-2
Original file line numberDiff line numberDiff line change
@@ -111,14 +111,13 @@ task yarnBuild(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
111111
outputs.dir('dist')
112112
}
113113

114-
task cleanExtraDirs {
114+
clean {
115115
delete 'node_modules/.yarn-integrity'
116116
delete 'dist'
117117
delete 'tmp'
118118
delete 'just'
119119
delete fileTree(dir: 'src', include: '*.generated.ts')
120120
}
121-
clean.finalizedBy(cleanExtraDirs)
122121

123122
configurations {
124123
assets

‎datahub-web-react/src/app/domain/DomainsList.tsx

+1
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,7 @@ export const DomainsList = () => {
196196
},
197197
ownership: null,
198198
entities: null,
199+
displayProperties: null,
199200
},
200201
pageSize,
201202
);

‎datahub-web-react/src/app/domain/utils.ts

+1
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ export const updateListDomainsCache = (
7272
children: null,
7373
dataProducts: null,
7474
parentDomains: null,
75+
displayProperties: null,
7576
},
7677
1000,
7778
parentDomain,

‎datahub-web-react/src/app/entity/mlModelGroup/profile/ModelGroupModels.tsx

+2-2
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,8 @@ export default function MLGroupModels() {
125125
},
126126
},
127127
{
128-
title: 'Tags',
129-
key: 'tags',
128+
title: 'Properties',
129+
key: 'properties',
130130
width: 200,
131131
render: (_: any, record: any) => {
132132
const tags = record.properties?.tags || [];

‎datahub-web-react/src/app/entity/shared/entityForm/FormByEntity.tsx

+7-6
Original file line numberDiff line numberDiff line change
@@ -63,13 +63,14 @@ export default function FormByEntity({ formUrn }: Props) {
6363
<ContentWrapper>
6464
<ProgressBar formUrn={formUrn} />
6565
<FlexWrapper>
66-
<ProfileSidebar
67-
sidebarSections={loading ? [] : sections}
68-
topSection={{ component: () => <EntityInfo formUrn={formUrn} /> }}
69-
backgroundColor="white"
70-
alignLeft
71-
/>
7266
<Form formUrn={formUrn} />
67+
{selectedEntityData && (
68+
<ProfileSidebar
69+
sidebarSections={loading ? [] : sections}
70+
topSection={{ component: () => <EntityInfo formUrn={formUrn} /> }}
71+
backgroundColor="white"
72+
/>
73+
)}
7374
</FlexWrapper>
7475
</ContentWrapper>
7576
</EntityContext.Provider>

‎datahub-web-react/src/app/entityV2/mlModelGroup/profile/ModelGroupModels.tsx

+2-2
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,8 @@ export default function MLGroupModels() {
128128
},
129129
},
130130
{
131-
title: 'Tags',
132-
key: 'tags',
131+
title: 'Properties',
132+
key: 'properties',
133133
width: 200,
134134
render: (_: any, record: any) => {
135135
const tags = record.properties?.tags || [];

‎datahub-web-react/src/app/entityV2/shared/EntityDropdown/UpdateDeprecationModal.tsx

+6-1
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,12 @@ export const UpdateDeprecationModal = ({ urns, resourceRefs, onClose, refetch, z
179179
</Button>
180180
)}
181181
{!replacementUrn && (
182-
<Button variant="outline" size="sm" onClick={() => setIsReplacementModalVisible(true)}>
182+
<Button
183+
variant="outline"
184+
type="button"
185+
size="sm"
186+
onClick={() => setIsReplacementModalVisible(true)}
187+
>
183188
Select Replacement
184189
</Button>
185190
)}

0 commit comments

Comments
 (0)