Skip to content

Commit 62f2789

Browse files
david-leifkerdavid-leifker
authored
feat(structured-properties): soft delete (#9812)
1 parent 60e4b2d commit 62f2789

File tree

143 files changed

+4726
-2748
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

143 files changed

+4726
-2748
lines changed

‎build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ plugins {
7373
id 'com.gorylenko.gradle-git-properties' version '2.4.1'
7474
id 'com.github.johnrengelman.shadow' version '8.1.1' apply false
7575
id 'com.palantir.docker' version '0.35.0' apply false
76-
id 'com.avast.gradle.docker-compose' version '0.17.5'
76+
id 'com.avast.gradle.docker-compose' version '0.17.6'
7777
id "com.diffplug.spotless" version "6.23.3"
7878
// https://blog.ltgt.net/javax-jakarta-mess-and-gradle-solution/
7979
// TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"

‎datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java

+10-10
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor;
1414
import com.linkedin.metadata.entity.EntityService;
1515
import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl;
16-
import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem;
16+
import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl;
1717
import com.linkedin.metadata.models.registry.ConfigEntityRegistry;
1818
import com.linkedin.metadata.models.registry.EntityRegistry;
1919
import com.linkedin.mxe.MetadataChangeProposal;
@@ -22,14 +22,14 @@
2222

2323
public class TestUtils {
2424

25-
public static EntityService<MCPUpsertBatchItem> getMockEntityService() {
25+
public static EntityService<ChangeItemImpl> getMockEntityService() {
2626
PathSpecBasedSchemaAnnotationVisitor.class
2727
.getClassLoader()
2828
.setClassAssertionStatus(PathSpecBasedSchemaAnnotationVisitor.class.getName(), false);
2929
EntityRegistry registry =
3030
new ConfigEntityRegistry(TestUtils.class.getResourceAsStream("/test-entity-registry.yaml"));
31-
EntityService<MCPUpsertBatchItem> mockEntityService =
32-
(EntityService<MCPUpsertBatchItem>) Mockito.mock(EntityService.class);
31+
EntityService<ChangeItemImpl> mockEntityService =
32+
(EntityService<ChangeItemImpl>) Mockito.mock(EntityService.class);
3333
Mockito.when(mockEntityService.getEntityRegistry()).thenReturn(registry);
3434
return mockEntityService;
3535
}
@@ -111,14 +111,14 @@ public static QueryContext getMockDenyContext(String actorUrn, AuthorizationRequ
111111
}
112112

113113
public static void verifyIngestProposal(
114-
EntityService<MCPUpsertBatchItem> mockService,
114+
EntityService<ChangeItemImpl> mockService,
115115
int numberOfInvocations,
116116
MetadataChangeProposal proposal) {
117117
verifyIngestProposal(mockService, numberOfInvocations, List.of(proposal));
118118
}
119119

120120
public static void verifyIngestProposal(
121-
EntityService<MCPUpsertBatchItem> mockService,
121+
EntityService<ChangeItemImpl> mockService,
122122
int numberOfInvocations,
123123
List<MetadataChangeProposal> proposals) {
124124
AspectsBatchImpl batch =
@@ -128,29 +128,29 @@ public static void verifyIngestProposal(
128128
}
129129

130130
public static void verifySingleIngestProposal(
131-
EntityService<MCPUpsertBatchItem> mockService,
131+
EntityService<ChangeItemImpl> mockService,
132132
int numberOfInvocations,
133133
MetadataChangeProposal proposal) {
134134
Mockito.verify(mockService, Mockito.times(numberOfInvocations))
135135
.ingestProposal(Mockito.eq(proposal), Mockito.any(AuditStamp.class), Mockito.eq(false));
136136
}
137137

138138
public static void verifyIngestProposal(
139-
EntityService<MCPUpsertBatchItem> mockService, int numberOfInvocations) {
139+
EntityService<ChangeItemImpl> mockService, int numberOfInvocations) {
140140
Mockito.verify(mockService, Mockito.times(numberOfInvocations))
141141
.ingestProposal(Mockito.any(AspectsBatchImpl.class), Mockito.eq(false));
142142
}
143143

144144
public static void verifySingleIngestProposal(
145-
EntityService<MCPUpsertBatchItem> mockService, int numberOfInvocations) {
145+
EntityService<ChangeItemImpl> mockService, int numberOfInvocations) {
146146
Mockito.verify(mockService, Mockito.times(numberOfInvocations))
147147
.ingestProposal(
148148
Mockito.any(MetadataChangeProposal.class),
149149
Mockito.any(AuditStamp.class),
150150
Mockito.eq(false));
151151
}
152152

153-
public static void verifyNoIngestProposal(EntityService<MCPUpsertBatchItem> mockService) {
153+
public static void verifyNoIngestProposal(EntityService<ChangeItemImpl> mockService) {
154154
Mockito.verify(mockService, Mockito.times(0))
155155
.ingestProposal(Mockito.any(AspectsBatchImpl.class), Mockito.anyBoolean());
156156
}

‎datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/AddTagsResolverTest.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils;
1919
import com.linkedin.metadata.entity.EntityService;
2020
import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl;
21-
import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem;
21+
import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl;
2222
import com.linkedin.mxe.MetadataChangeProposal;
2323
import graphql.schema.DataFetchingEnvironment;
2424
import java.util.concurrent.CompletionException;
@@ -221,7 +221,7 @@ public void testGetUnauthorized() throws Exception {
221221

222222
@Test
223223
public void testGetEntityClientException() throws Exception {
224-
EntityService<MCPUpsertBatchItem> mockService = getMockEntityService();
224+
EntityService<ChangeItemImpl> mockService = getMockEntityService();
225225

226226
Mockito.doThrow(RuntimeException.class)
227227
.when(mockService)

‎datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -178,8 +178,9 @@ private void readerExecutable(ReaderWrapper reader, UpgradeContext context) {
178178
final RecordTemplate aspectRecord;
179179
try {
180180
aspectRecord =
181-
EntityUtils.toAspectRecord(
182-
entityName, aspectName, aspect.getMetadata(), _entityRegistry);
181+
EntityUtils.toSystemAspect(aspect.toEntityAspect(), _entityService)
182+
.get()
183+
.getRecordTemplate();
183184
} catch (Exception e) {
184185
context
185186
.report()

‎datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java

+39-19
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,13 @@
22

33
import static com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils.INDEX_BLOCKS_WRITE_SETTING;
44
import static com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils.getAllReindexConfigs;
5+
import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME;
56
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME;
67
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME;
78

9+
import com.datahub.util.RecordUtils;
810
import com.google.common.collect.ImmutableMap;
11+
import com.linkedin.common.Status;
912
import com.linkedin.datahub.upgrade.UpgradeContext;
1013
import com.linkedin.datahub.upgrade.UpgradeStep;
1114
import com.linkedin.datahub.upgrade.UpgradeStepResult;
@@ -14,14 +17,15 @@
1417
import com.linkedin.gms.factory.config.ConfigurationProvider;
1518
import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory;
1619
import com.linkedin.metadata.entity.AspectDao;
17-
import com.linkedin.metadata.entity.EntityUtils;
1820
import com.linkedin.metadata.models.registry.EntityRegistry;
1921
import com.linkedin.metadata.search.elasticsearch.indexbuilder.ReindexConfig;
2022
import com.linkedin.metadata.shared.ElasticSearchIndexed;
2123
import com.linkedin.structured.StructuredPropertyDefinition;
24+
import com.linkedin.util.Pair;
2225
import java.io.IOException;
2326
import java.util.List;
2427
import java.util.Map;
28+
import java.util.Set;
2529
import java.util.function.Function;
2630
import java.util.stream.Collectors;
2731
import lombok.RequiredArgsConstructor;
@@ -54,24 +58,13 @@ public int retryCount() {
5458
public Function<UpgradeContext, UpgradeStepResult> executable() {
5559
return (context) -> {
5660
try {
57-
List<ReindexConfig> reindexConfigs =
58-
_configurationProvider.getStructuredProperties().isSystemUpdateEnabled()
59-
? getAllReindexConfigs(
60-
_services,
61-
_aspectDao
62-
.streamAspects(
63-
STRUCTURED_PROPERTY_ENTITY_NAME,
64-
STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)
65-
.map(
66-
entityAspect ->
67-
EntityUtils.toAspectRecord(
68-
STRUCTURED_PROPERTY_ENTITY_NAME,
69-
STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
70-
entityAspect.getMetadata(),
71-
_entityRegistry))
72-
.map(recordTemplate -> (StructuredPropertyDefinition) recordTemplate)
73-
.collect(Collectors.toSet()))
74-
: getAllReindexConfigs(_services);
61+
final List<ReindexConfig> reindexConfigs;
62+
if (_configurationProvider.getStructuredProperties().isSystemUpdateEnabled()) {
63+
reindexConfigs =
64+
getAllReindexConfigs(_services, getActiveStructuredPropertiesDefinitions(_aspectDao));
65+
} else {
66+
reindexConfigs = getAllReindexConfigs(_services);
67+
}
7568

7669
// Get indices to update
7770
List<ReindexConfig> indexConfigs =
@@ -160,4 +153,31 @@ private boolean blockWrites(String indexName) throws InterruptedException, IOExc
160153

161154
return ack;
162155
}
156+
157+
private static Set<StructuredPropertyDefinition> getActiveStructuredPropertiesDefinitions(
158+
AspectDao aspectDao) {
159+
Set<String> removedStructuredPropertyUrns =
160+
aspectDao
161+
.streamAspects(STRUCTURED_PROPERTY_ENTITY_NAME, STATUS_ASPECT_NAME)
162+
.map(
163+
entityAspect ->
164+
Pair.of(
165+
entityAspect.getUrn(),
166+
RecordUtils.toRecordTemplate(Status.class, entityAspect.getMetadata())))
167+
.filter(status -> status.getSecond().isRemoved())
168+
.map(Pair::getFirst)
169+
.collect(Collectors.toSet());
170+
171+
return aspectDao
172+
.streamAspects(STRUCTURED_PROPERTY_ENTITY_NAME, STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)
173+
.map(
174+
entityAspect ->
175+
Pair.of(
176+
entityAspect.getUrn(),
177+
RecordUtils.toRecordTemplate(
178+
StructuredPropertyDefinition.class, entityAspect.getMetadata())))
179+
.filter(definition -> !removedStructuredPropertyUrns.contains(definition.getKey()))
180+
.map(Pair::getSecond)
181+
.collect(Collectors.toSet());
182+
}
163183
}

‎datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNoSchemaRegistryTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public void testSystemUpdateInit() {
4949
@Test
5050
public void testSystemUpdateKafkaProducerOverride() {
5151
assertEquals(kafkaEventProducer, duheKafkaEventProducer);
52-
assertEquals(entityService.get_producer(), duheKafkaEventProducer);
52+
assertEquals(entityService.getProducer(), duheKafkaEventProducer);
5353
}
5454

5555
@Test

‎docs/api/openapi/openapi-structured-properties.md

+44
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,50 @@ Example Response:
8686

8787
### Delete Property Definition
8888

89+
There are two types of deletion present in DataHub: `hard` and `soft` delete. As of the current release only the `soft` delete
90+
is supported for Structured Properties. See the subsections below for more details.
91+
92+
#### Soft Delete
93+
94+
A `soft` deleted Structured Property does not remove any underlying data on the Structured Property entity
95+
or the Structured Property's values written to other entities. The `soft` delete is 100% reversible with zero data loss.
96+
When a Structured Property is `soft` deleted, a few operations are not available.
97+
98+
Structured Property Soft Delete Effects:
99+
100+
* Entities with a `soft` deleted Structured Property value will not return the `soft` deleted properties
101+
* Updates to a `soft` deleted Structured Property's definition are denied
102+
* Adding a `soft` deleted Structured Property's value to an entity is denied
103+
* Search filters using a `soft` deleted Structured Property will be denied
104+
105+
The following command will `soft` delete the test property `MyProperty01` created in this guide by writing
106+
to the `status` aspect.
107+
108+
```shell
109+
curl -X 'POST' \
110+
'http://localhost:8080/openapi/v2/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Amy.test.MyProperty01/status?systemMetadata=false' \
111+
-H 'accept: application/json' \
112+
-H 'Content-Type: application/json' \
113+
-d '{
114+
"removed": true
115+
}' | jq
116+
```
117+
118+
Removing the `soft` delete from the Structured Property can be done by either `hard` deleting the `status` aspect or
119+
changing the `removed` boolean to `false.
120+
121+
```shell
122+
curl -X 'POST' \
123+
'http://localhost:8080/openapi/v2/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Amy.test.MyProperty01/status?systemMetadata=false' \
124+
-H 'accept: application/json' \
125+
-H 'Content-Type: application/json' \
126+
-d '{
127+
"removed": false
128+
}' | jq
129+
```
130+
131+
#### Hard Delete
132+
89133
⚠ **Not Implemented** ⚠
90134

91135
## Applying Structured Properties

‎entity-registry/build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ dependencies {
3232
testImplementation externalDependency.mockito
3333
testImplementation externalDependency.mockitoInline
3434
testCompileOnly externalDependency.lombok
35+
testAnnotationProcessor externalDependency.lombok
3536
testImplementation externalDependency.classGraph
3637

3738
}

‎entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectRetriever.java ‎entity-registry/src/main/java/com/linkedin/metadata/aspect/AspectRetriever.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package com.linkedin.metadata.aspect.plugins.validation;
1+
package com.linkedin.metadata.aspect;
22

33
import com.google.common.collect.ImmutableSet;
44
import com.linkedin.common.urn.Urn;

‎entity-registry/src/main/java/com/linkedin/metadata/aspect/CachingAspectRetriever.java

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
package com.linkedin.metadata.aspect;
2+
3+
/** Responses can be cached based on application.yaml caching configuration for the EntityClient */
4+
public interface CachingAspectRetriever extends AspectRetriever {}

‎entity-registry/src/main/java/com/linkedin/metadata/aspect/ReadItem.java

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package com.linkedin.metadata.aspect;
2+
3+
import com.linkedin.common.urn.Urn;
4+
import com.linkedin.data.DataMap;
5+
import com.linkedin.data.template.RecordTemplate;
6+
import com.linkedin.metadata.models.AspectSpec;
7+
import com.linkedin.metadata.models.EntitySpec;
8+
import com.linkedin.mxe.SystemMetadata;
9+
import java.lang.reflect.InvocationTargetException;
10+
import javax.annotation.Nonnull;
11+
import javax.annotation.Nullable;
12+
13+
public interface ReadItem {
14+
/**
15+
* The urn associated with the aspect
16+
*
17+
* @return
18+
*/
19+
@Nonnull
20+
Urn getUrn();
21+
22+
/**
23+
* Aspect's name
24+
*
25+
* @return the name
26+
*/
27+
@Nonnull
28+
default String getAspectName() {
29+
return getAspectSpec().getName();
30+
}
31+
32+
@Nullable
33+
RecordTemplate getRecordTemplate();
34+
35+
default <T> T getAspect(Class<T> clazz) {
36+
return getAspect(clazz, getRecordTemplate());
37+
}
38+
39+
static <T> T getAspect(Class<T> clazz, @Nullable RecordTemplate recordTemplate) {
40+
if (recordTemplate != null) {
41+
try {
42+
return clazz.getConstructor(DataMap.class).newInstance(recordTemplate.data());
43+
} catch (InstantiationException
44+
| IllegalAccessException
45+
| InvocationTargetException
46+
| NoSuchMethodException e) {
47+
throw new RuntimeException(e);
48+
}
49+
} else {
50+
return null;
51+
}
52+
}
53+
54+
/**
55+
* System information
56+
*
57+
* @return the system metadata
58+
*/
59+
@Nullable
60+
SystemMetadata getSystemMetadata();
61+
62+
/**
63+
* The entity's schema
64+
*
65+
* @return entity specification
66+
*/
67+
@Nonnull
68+
EntitySpec getEntitySpec();
69+
70+
/**
71+
* The aspect's schema
72+
*
73+
* @return aspect's specification
74+
*/
75+
@Nonnull
76+
AspectSpec getAspectSpec();
77+
}

‎entity-registry/src/main/java/com/linkedin/metadata/aspect/SystemAspect.java

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package com.linkedin.metadata.aspect;
2+
3+
import com.linkedin.common.AuditStamp;
4+
import com.linkedin.common.urn.UrnUtils;
5+
import java.sql.Timestamp;
6+
import javax.annotation.Nonnull;
7+
8+
/**
9+
* An aspect along with system metadata and creation timestamp. Represents an aspect as stored in
10+
* primary storage.
11+
*/
12+
public interface SystemAspect extends ReadItem {
13+
long getVersion();
14+
15+
Timestamp getCreatedOn();
16+
17+
String getCreatedBy();
18+
19+
@Nonnull
20+
default AuditStamp getAuditStamp() {
21+
return new AuditStamp()
22+
.setActor(UrnUtils.getUrn(getCreatedBy()))
23+
.setTime(getCreatedOn().getTime());
24+
}
25+
}

0 commit comments

Comments
 (0)