Skip to content

Commit bed1880

Browse files
sid-acrylsid-acryl
committed
merge master
2 parents 6def894 + 2e54461 commit bed1880

File tree

204 files changed

+7215
-3346
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

204 files changed

+7215
-3346
lines changed

‎.github/workflows/check-datahub-jars.yml

+12-5
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,12 @@ on:
55
branches:
66
- master
77
paths:
8-
- "metadata-integration"
8+
- "metadata-integration/**"
99
pull_request:
1010
branches:
1111
- "**"
1212
paths:
13-
- "metadata-integration"
13+
- "metadata-integration/**"
1414
release:
1515
types: [published]
1616

@@ -28,15 +28,22 @@ jobs:
2828
runs-on: ubuntu-latest
2929
steps:
3030
- uses: acryldata/sane-checkout-action@v3
31+
- uses: actions/setup-python@v5
32+
with:
33+
python-version: "3.10"
34+
- uses: actions/cache@v4
35+
with:
36+
path: |
37+
~/.cache/uv
38+
key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
39+
- name: Install dependencies
40+
run: ./metadata-ingestion/scripts/install_deps.sh
3141
- name: Set up JDK 17
3242
uses: actions/setup-java@v4
3343
with:
3444
distribution: "zulu"
3545
java-version: 17
3646
- uses: gradle/actions/setup-gradle@v3
37-
- uses: actions/setup-python@v5
38-
with:
39-
python-version: "3.10"
4047
- name: check ${{ matrix.command }} jar
4148
run: |
4249
./gradlew :metadata-integration:java:${{ matrix.command }}:build --info

‎build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,7 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
373373
exclude group: "org.slf4j", module: "slf4j-log4j12"
374374
exclude group: "org.slf4j", module: "slf4j-nop"
375375
exclude group: "org.slf4j", module: "slf4j-ext"
376+
exclude group: "org.codehaus.jackson", module: "jackson-mapper-asl"
376377

377378
resolutionStrategy.force externalDependency.antlr4Runtime
378379
resolutionStrategy.force externalDependency.antlr4

‎datahub-frontend/app/auth/AuthModule.java

+2
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import io.datahubproject.metadata.context.EntityRegistryContext;
2828
import io.datahubproject.metadata.context.OperationContext;
2929
import io.datahubproject.metadata.context.OperationContextConfig;
30+
import io.datahubproject.metadata.context.RetrieverContext;
3031
import io.datahubproject.metadata.context.SearchContext;
3132
import io.datahubproject.metadata.context.ValidationContext;
3233
import java.nio.charset.StandardCharsets;
@@ -195,6 +196,7 @@ protected OperationContext provideOperationContext(
195196
.searchContext(SearchContext.EMPTY)
196197
.entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY))
197198
.validationContext(ValidationContext.builder().alternateValidation(false).build())
199+
.retrieverContext(RetrieverContext.EMPTY)
198200
.build(systemAuthentication);
199201
}
200202

‎datahub-frontend/conf/logback.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@
6161
<logger name="com.linkedin" level="DEBUG">
6262
<appender-ref ref="DEBUG_FILE"/>
6363
</logger>
64-
<logger name="controller" level="DEBUG">
64+
<logger name="controllers" level="DEBUG">
6565
<appender-ref ref="DEBUG_FILE"/>
6666
</logger>
6767
<logger name="auth" level="DEBUG">

‎datahub-upgrade/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ dependencies {
6060
// mock internal schema registry
6161
implementation externalDependency.kafkaAvroSerde
6262
implementation externalDependency.kafkaAvroSerializer
63-
implementation "org.apache.kafka:kafka_2.12:3.7.1"
63+
implementation "org.apache.kafka:kafka_2.13:3.7.2"
6464

6565
implementation externalDependency.slf4jApi
6666
compileOnly externalDependency.lombok

‎datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java

+2
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import com.linkedin.gms.factory.kafka.common.TopicConventionFactory;
1414
import com.linkedin.gms.factory.kafka.schemaregistry.InternalSchemaRegistryFactory;
1515
import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory;
16+
import com.linkedin.metadata.aspect.CachingAspectRetriever;
1617
import com.linkedin.metadata.config.kafka.KafkaConfiguration;
1718
import com.linkedin.metadata.dao.producer.KafkaEventProducer;
1819
import com.linkedin.metadata.dao.producer.KafkaHealthChecker;
@@ -186,6 +187,7 @@ protected OperationContext javaSystemOperationContext(
186187
components.getIndexConvention(),
187188
RetrieverContext.builder()
188189
.aspectRetriever(entityServiceAspectRetriever)
190+
.cachingAspectRetriever(CachingAspectRetriever.EMPTY)
189191
.graphRetriever(systemGraphRetriever)
190192
.searchRetriever(searchServiceSearchRetriever)
191193
.build(),

‎datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ private void readerExecutable(ReaderWrapper reader, UpgradeContext context) {
180180
try {
181181
aspectRecord =
182182
EntityUtils.toSystemAspect(
183-
context.opContext().getRetrieverContext().get(), aspect.toEntityAspect())
183+
context.opContext().getRetrieverContext(), aspect.toEntityAspect())
184184
.get()
185185
.getRecordTemplate();
186186
} catch (Exception e) {

‎datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,7 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
113113
List<Pair<Future<?>, SystemAspect>> futures;
114114
futures =
115115
EntityUtils.toSystemAspectFromEbeanAspects(
116-
opContext.getRetrieverContext().get(),
117-
batch.collect(Collectors.toList()))
116+
opContext.getRetrieverContext(), batch.collect(Collectors.toList()))
118117
.stream()
119118
.map(
120119
systemAspect -> {

‎datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtil.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -100,8 +100,8 @@ static AspectsBatch generateAspectBatch(
100100
.collect(Collectors.toList());
101101

102102
return AspectsBatchImpl.builder()
103-
.mcps(mcps, auditStamp, opContext.getRetrieverContext().get())
104-
.retrieverContext(opContext.getRetrieverContext().get())
103+
.mcps(mcps, auditStamp, opContext.getRetrieverContext())
104+
.retrieverContext(opContext.getRetrieverContext())
105105
.build();
106106
}
107107

‎datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/dataprocessinstances/BackfillDataProcessInstancesHasRunEventsStep.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
import static com.linkedin.metadata.Constants.*;
44

5+
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
6+
import com.fasterxml.jackson.databind.node.ObjectNode;
57
import com.google.common.base.Throwables;
68
import com.linkedin.common.urn.Urn;
79
import com.linkedin.datahub.upgrade.UpgradeContext;
@@ -23,8 +25,6 @@
2325
import java.util.Set;
2426
import java.util.function.Function;
2527
import lombok.extern.slf4j.Slf4j;
26-
import org.codehaus.jackson.node.JsonNodeFactory;
27-
import org.codehaus.jackson.node.ObjectNode;
2828
import org.opensearch.action.search.SearchRequest;
2929
import org.opensearch.action.search.SearchResponse;
3030
import org.opensearch.client.RequestOptions;

‎datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadataStep.java

+3-7
Original file line numberDiff line numberDiff line change
@@ -168,13 +168,13 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
168168

169169
AspectsBatch aspectsBatch =
170170
AspectsBatchImpl.builder()
171-
.retrieverContext(opContext.getRetrieverContext().get())
171+
.retrieverContext(opContext.getRetrieverContext())
172172
.items(
173173
batch
174174
.flatMap(
175175
ebeanAspectV2 ->
176176
EntityUtils.toSystemAspectFromEbeanAspects(
177-
opContext.getRetrieverContext().get(),
177+
opContext.getRetrieverContext(),
178178
Set.of(ebeanAspectV2))
179179
.stream())
180180
.map(
@@ -189,11 +189,7 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
189189
.auditStamp(systemAspect.getAuditStamp())
190190
.systemMetadata(
191191
withAppSource(systemAspect.getSystemMetadata()))
192-
.build(
193-
opContext
194-
.getRetrieverContext()
195-
.get()
196-
.getAspectRetriever()))
192+
.build(opContext.getAspectRetriever()))
197193
.collect(Collectors.toList()))
198194
.build();
199195

‎datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/schemafield/GenerateSchemaFieldsFromSchemaMetadataStepTest.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import com.linkedin.upgrade.DataHubUpgradeState;
2323
import io.datahubproject.metadata.context.OperationContext;
2424
import io.datahubproject.metadata.context.RetrieverContext;
25-
import java.util.Optional;
2625
import java.util.stream.Stream;
2726
import org.junit.jupiter.api.BeforeEach;
2827
import org.junit.jupiter.api.Test;
@@ -48,7 +47,7 @@ public void setup() {
4847
step =
4948
new GenerateSchemaFieldsFromSchemaMetadataStep(
5049
mockOpContext, mockEntityService, mockAspectDao, 10, 100, 1000);
51-
when(mockOpContext.getRetrieverContext()).thenReturn(Optional.of(mockRetrieverContext));
50+
when(mockOpContext.getRetrieverContext()).thenReturn(mockRetrieverContext);
5251
}
5352

5453
/** Test to verify the correct step ID is returned. */

‎datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx

+2-4
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,9 @@ export class SchemaFieldPropertiesEntity implements Entity<SchemaFieldEntity> {
3535
// Currently unused.
3636
getPathName = () => 'schemaField';
3737

38-
// Currently unused.
39-
getEntityName = () => 'schemaField';
38+
getEntityName = () => 'Column';
4039

41-
// Currently unused.
42-
getCollectionName = () => 'schemaFields';
40+
getCollectionName = () => 'Columns';
4341

4442
// Currently unused.
4543
renderProfile = (_: string) => <></>;

‎datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx

-2
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@ const DeprecatedContainer = styled.div`
1919
justify-content: center;
2020
align-items: center;
2121
color: #cd0d24;
22-
margin-left: 0px;
23-
margin-right: 8px;
2422
padding-top: 8px;
2523
padding-bottom: 8px;
2624
padding-right: 4px;

‎datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/MultipleStringInput.tsx ‎datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/MultipleOpenEndedInput.tsx

+8-5
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import React from 'react';
44
import styled from 'styled-components';
55
import { ANTD_GRAY_V2 } from '../../../constants';
66

7+
const MultiStringWrapper = styled.div``;
8+
79
const StyledInput = styled(Input)`
810
width: 75%;
911
min-width: 350px;
@@ -29,10 +31,11 @@ const DeleteButton = styled(Button)`
2931

3032
interface Props {
3133
selectedValues: any[];
34+
inputType?: string;
3235
updateSelectedValues: (values: any[]) => void;
3336
}
3437

35-
export default function MultipleStringInput({ selectedValues, updateSelectedValues }: Props) {
38+
export default function MultipleOpenEndedInput({ selectedValues, updateSelectedValues, inputType = 'text' }: Props) {
3639
function updateInput(text: string, index: number) {
3740
const updatedValues =
3841
selectedValues.length > 0 ? selectedValues.map((value, i) => (i === index ? text : value)) : [text];
@@ -53,14 +56,14 @@ export default function MultipleStringInput({ selectedValues, updateSelectedValu
5356
}
5457

5558
return (
56-
<div>
59+
<MultiStringWrapper>
5760
{selectedValues.length > 1 &&
5861
selectedValues.map((selectedValue, index) => {
5962
const key = `${index}`;
6063
return (
6164
<InputWrapper key={key}>
6265
<StyledInput
63-
type="text"
66+
type={inputType}
6467
value={selectedValue}
6568
onChange={(e) => updateInput(e.target.value, index)}
6669
/>
@@ -70,14 +73,14 @@ export default function MultipleStringInput({ selectedValues, updateSelectedValu
7073
})}
7174
{selectedValues.length <= 1 && (
7275
<StyledInput
73-
type="text"
76+
type={inputType}
7477
value={selectedValues[0] || ''}
7578
onChange={(e) => updateInput(e.target.value, 0)}
7679
/>
7780
)}
7881
<StyledButton type="link" onClick={addNewValue}>
7982
+ Add More
8083
</StyledButton>
81-
</div>
84+
</MultiStringWrapper>
8285
);
8386
}

‎datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/NumberInput.tsx

+19-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
import { Input } from 'antd';
22
import React, { ChangeEvent } from 'react';
33
import styled from 'styled-components';
4+
import { PropertyCardinality } from '@src/types.generated';
45
import { ANTD_GRAY_V2 } from '../../../constants';
6+
import MultipleOpenEndedInput from './MultipleOpenEndedInput';
57

68
const StyledInput = styled(Input)`
79
border: 1px solid ${ANTD_GRAY_V2[6]};
@@ -10,15 +12,31 @@ const StyledInput = styled(Input)`
1012

1113
interface Props {
1214
selectedValues: any[];
15+
cardinality?: PropertyCardinality | null;
1316
updateSelectedValues: (values: string[] | number[]) => void;
1417
}
1518

16-
export default function NumberInput({ selectedValues, updateSelectedValues }: Props) {
19+
export default function NumberInput({ selectedValues, cardinality, updateSelectedValues }: Props) {
1720
function updateInput(event: ChangeEvent<HTMLInputElement>) {
1821
const number = Number(event.target.value);
1922
updateSelectedValues([number]);
2023
}
2124

25+
function updateMultipleValues(values: string[] | number[]) {
26+
const numbers = values.map((v) => Number(v));
27+
updateSelectedValues(numbers);
28+
}
29+
30+
if (cardinality === PropertyCardinality.Multiple) {
31+
return (
32+
<MultipleOpenEndedInput
33+
selectedValues={selectedValues}
34+
updateSelectedValues={updateMultipleValues}
35+
inputType="number"
36+
/>
37+
);
38+
}
39+
2240
return (
2341
<StyledInput
2442
type="number"

‎datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/StringInput.tsx

+10-3
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import React, { ChangeEvent } from 'react';
33
import styled from 'styled-components';
44
import { ANTD_GRAY_V2 } from '../../../constants';
55
import { PropertyCardinality } from '../../../../../../types.generated';
6-
import MultipleStringInput from './MultipleStringInput';
6+
import MultipleOpenEndedInput from './MultipleOpenEndedInput';
77

88
const StyledInput = styled(Input)`
99
width: 75%;
@@ -24,8 +24,15 @@ export default function StringInput({ selectedValues, cardinality, updateSelecte
2424
}
2525

2626
if (cardinality === PropertyCardinality.Multiple) {
27-
return <MultipleStringInput selectedValues={selectedValues} updateSelectedValues={updateSelectedValues} />;
27+
return <MultipleOpenEndedInput selectedValues={selectedValues} updateSelectedValues={updateSelectedValues} />;
2828
}
2929

30-
return <StyledInput type="text" value={selectedValues[0] || ''} onChange={updateInput} />;
30+
return (
31+
<StyledInput
32+
type="text"
33+
value={selectedValues[0] || ''}
34+
onChange={updateInput}
35+
data-testid="structured-property-string-value-input"
36+
/>
37+
);
3138
}

‎datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/StructuredPropertyInput.tsx

+5-1
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,11 @@ export default function StructuredPropertyInput({
6060
<DateInput selectedValues={selectedValues} updateSelectedValues={updateSelectedValues} />
6161
)}
6262
{!allowedValues && valueType.info.type === StdDataType.Number && (
63-
<NumberInput selectedValues={selectedValues} updateSelectedValues={updateSelectedValues} />
63+
<NumberInput
64+
selectedValues={selectedValues}
65+
cardinality={cardinality}
66+
updateSelectedValues={updateSelectedValues}
67+
/>
6468
)}
6569
{!allowedValues && valueType.info.type === StdDataType.Urn && (
6670
<UrnInput

‎datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ const TitleWrapper = styled.div`
2424
display: flex;
2525
justify-content: left;
2626
align-items: center;
27+
gap: 8px;
2728
2829
.ant-typography-edit-content {
2930
padding-top: 7px;

‎datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx

-2
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@ import { useEntityData, useRefetch } from '../../../EntityContext';
88
import { useGlossaryEntityData } from '../../../GlossaryEntityContext';
99

1010
export const EntityTitle = styled(Typography.Title)`
11-
margin-right: 10px;
12-
1311
&&& {
1412
margin-bottom: 0;
1513
word-break: break-all;

0 commit comments

Comments
 (0)