Skip to content

Commit 41b0629

Browse files
david-leifkerdavid-leifker
authored
feat(api): URN, Entity, and Aspect name Async Validation (#12797)
1 parent 2bc1e52 commit 41b0629

File tree

22 files changed

+1165
-211
lines changed

22 files changed

+1165
-211
lines changed

‎docs/how/updating-datahub.md

+2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
3030

3131
- #12716: Fix the `platform_instance` being added twice to the URN. If you want to have the previous behavior back, you need to add your platform_instance twice (i.e. `plat.plat`).
3232

33+
- #12797: Previously endpoints when used in ASYNC mode would not validate URNs, entity & aspect names immediately. Starting with this release, even in ASYNC mode, these requests will be returned with http code 400.
34+
3335

3436
### Known Issues
3537

‎entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/MCPItem.java

+4
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package com.linkedin.metadata.aspect.batch;
22

33
import com.google.common.collect.ImmutableSet;
4+
import com.linkedin.common.urn.Urn;
45
import com.linkedin.events.metadata.ChangeType;
56
import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine;
67
import com.linkedin.metadata.models.AspectSpec;
8+
import com.linkedin.metadata.models.registry.EntityRegistry;
79
import com.linkedin.mxe.MetadataChangeProposal;
810
import com.linkedin.mxe.SystemMetadata;
911
import java.util.Collections;
@@ -82,4 +84,6 @@ static boolean supportsPatch(AspectSpec aspectSpec) {
8284
}
8385
return true;
8486
}
87+
88+
default void validate(Urn urn, String aspectName, EntityRegistry entityRegistry) {}
8589
}

‎metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java

+14-18
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import com.linkedin.metadata.aspect.plugins.hooks.MutationHook;
1515
import com.linkedin.metadata.aspect.plugins.validation.ValidationExceptionCollection;
1616
import com.linkedin.metadata.entity.validation.ValidationException;
17-
import com.linkedin.metadata.models.EntitySpec;
1817
import com.linkedin.mxe.MetadataChangeProposal;
1918
import com.linkedin.util.Pair;
2019
import java.util.ArrayList;
@@ -153,10 +152,11 @@ private Stream<? extends BatchItem> proposedItemsToChangeItemStream(List<MCPItem
153152

154153
private static BatchItem patchDiscriminator(MCPItem mcpItem, AspectRetriever aspectRetriever) {
155154
if (ChangeType.PATCH.equals(mcpItem.getChangeType())) {
156-
return PatchItemImpl.PatchItemImplBuilder.build(
157-
mcpItem.getMetadataChangeProposal(),
158-
mcpItem.getAuditStamp(),
159-
aspectRetriever.getEntityRegistry());
155+
return PatchItemImpl.builder()
156+
.build(
157+
mcpItem.getMetadataChangeProposal(),
158+
mcpItem.getAuditStamp(),
159+
aspectRetriever.getEntityRegistry());
160160
}
161161
return ChangeItemImpl.builder()
162162
.build(mcpItem.getMetadataChangeProposal(), mcpItem.getAuditStamp(), aspectRetriever);
@@ -195,22 +195,18 @@ public AspectsBatchImplBuilder mcps(
195195
mcp -> {
196196
try {
197197
if (alternateMCPValidation) {
198-
EntitySpec entitySpec =
199-
retrieverContext
200-
.getAspectRetriever()
201-
.getEntityRegistry()
202-
.getEntitySpec(mcp.getEntityType());
203198
return ProposedItem.builder()
204-
.metadataChangeProposal(mcp)
205-
.entitySpec(entitySpec)
206-
.auditStamp(auditStamp)
207-
.build();
199+
.build(
200+
mcp,
201+
auditStamp,
202+
retrieverContext.getAspectRetriever().getEntityRegistry());
208203
}
209204
if (mcp.getChangeType().equals(ChangeType.PATCH)) {
210-
return PatchItemImpl.PatchItemImplBuilder.build(
211-
mcp,
212-
auditStamp,
213-
retrieverContext.getAspectRetriever().getEntityRegistry());
205+
return PatchItemImpl.builder()
206+
.build(
207+
mcp,
208+
auditStamp,
209+
retrieverContext.getAspectRetriever().getEntityRegistry());
214210
} else {
215211
return ChangeItemImpl.builder()
216212
.build(mcp, auditStamp, retrieverContext.getAspectRetriever());

‎metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImpl.java

+37-39
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
import com.linkedin.metadata.aspect.batch.ChangeMCP;
1414
import com.linkedin.metadata.aspect.batch.MCPItem;
1515
import com.linkedin.metadata.aspect.patch.template.common.GenericPatchTemplate;
16-
import com.linkedin.metadata.entity.AspectUtils;
1716
import com.linkedin.metadata.entity.validation.ValidationApiUtils;
1817
import com.linkedin.metadata.models.AspectSpec;
1918
import com.linkedin.metadata.models.EntitySpec;
@@ -139,38 +138,51 @@ public ChangeItemImplBuilder systemMetadata(SystemMetadata systemMetadata) {
139138
return this;
140139
}
141140

141+
public ChangeItemImplBuilder changeType(ChangeType changeType) {
142+
this.changeType = validateOrDefaultChangeType(changeType);
143+
return this;
144+
}
145+
142146
@SneakyThrows
143147
public ChangeItemImpl build(AspectRetriever aspectRetriever) {
144-
// Apply change type default
145-
this.changeType = validateOrDefaultChangeType(changeType);
148+
if (this.changeType == null) {
149+
changeType(null); // Apply change type default
150+
}
146151

147152
// Apply empty headers
148153
if (this.headers == null) {
149154
this.headers = Map.of();
150155
}
151156

152-
if (this.urn == null && this.metadataChangeProposal != null) {
153-
this.urn = this.metadataChangeProposal.getEntityUrn();
154-
}
155-
156157
ValidationApiUtils.validateUrn(aspectRetriever.getEntityRegistry(), this.urn);
157158
log.debug("entity type = {}", this.urn.getEntityType());
158159

159-
entitySpec(aspectRetriever.getEntityRegistry().getEntitySpec(this.urn.getEntityType()));
160+
entitySpec(
161+
ValidationApiUtils.validateEntity(
162+
aspectRetriever.getEntityRegistry(), this.urn.getEntityType()));
160163
log.debug("entity spec = {}", this.entitySpec);
161164

162-
aspectSpec(ValidationApiUtils.validate(this.entitySpec, this.aspectName));
165+
aspectSpec(ValidationApiUtils.validateAspect(this.entitySpec, this.aspectName));
163166
log.debug("aspect spec = {}", this.aspectSpec);
164167

168+
if (this.recordTemplate == null && this.metadataChangeProposal != null) {
169+
this.recordTemplate = convertToRecordTemplate(this.metadataChangeProposal, aspectSpec);
170+
}
171+
165172
ValidationApiUtils.validateRecordTemplate(
166173
this.entitySpec, this.urn, this.recordTemplate, aspectRetriever);
167174

175+
if (this.systemMetadata == null) {
176+
// generate default
177+
systemMetadata(null);
178+
}
179+
168180
return new ChangeItemImpl(
169181
this.changeType,
170182
this.urn,
171183
this.aspectName,
172184
this.recordTemplate,
173-
SystemMetadataUtils.generateSystemMetadataIfEmpty(this.systemMetadata),
185+
this.systemMetadata,
174186
this.auditStamp,
175187
this.metadataChangeProposal,
176188
this.entitySpec,
@@ -183,35 +195,16 @@ public ChangeItemImpl build(AspectRetriever aspectRetriever) {
183195
public ChangeItemImpl build(
184196
MetadataChangeProposal mcp, AuditStamp auditStamp, AspectRetriever aspectRetriever) {
185197

186-
log.debug("entity type = {}", mcp.getEntityType());
187-
EntitySpec entitySpec =
188-
aspectRetriever.getEntityRegistry().getEntitySpec(mcp.getEntityType());
189-
AspectSpec aspectSpec = AspectUtils.validateAspect(mcp, entitySpec);
190-
191-
if (!MCPItem.isValidChangeType(ChangeType.UPSERT, aspectSpec)) {
192-
throw new UnsupportedOperationException(
193-
"ChangeType not supported: "
194-
+ mcp.getChangeType()
195-
+ " for aspect "
196-
+ mcp.getAspectName());
197-
}
198-
199-
Urn urn = mcp.getEntityUrn();
200-
if (urn == null) {
201-
urn = EntityKeyUtils.getUrnFromProposal(mcp, entitySpec.getKeyAspectSpec());
202-
}
203-
204-
return ChangeItemImpl.builder()
205-
.changeType(mcp.getChangeType())
206-
.urn(urn)
207-
.aspectName(mcp.getAspectName())
208-
.systemMetadata(
209-
SystemMetadataUtils.generateSystemMetadataIfEmpty(mcp.getSystemMetadata()))
210-
.metadataChangeProposal(mcp)
211-
.auditStamp(auditStamp)
212-
.recordTemplate(convertToRecordTemplate(mcp, aspectSpec))
213-
.nextAspectVersion(this.nextAspectVersion)
214-
.build(aspectRetriever);
198+
// Validation includes: Urn, Entity, Aspect
199+
this.metadataChangeProposal =
200+
ValidationApiUtils.validateMCP(aspectRetriever.getEntityRegistry(), mcp);
201+
this.urn = this.metadataChangeProposal.getEntityUrn(); // validation ensures existence
202+
this.auditStamp = auditStamp;
203+
this.aspectName = mcp.getAspectName(); // prior validation
204+
changeType(mcp.getChangeType());
205+
this.systemMetadata = mcp.getSystemMetadata();
206+
this.headers = mcp.getHeaders();
207+
return build(aspectRetriever);
215208
}
216209

217210
// specific to impl, other impls support PATCH, etc
@@ -226,6 +219,11 @@ private static ChangeType validateOrDefaultChangeType(@Nullable ChangeType chang
226219

227220
private static RecordTemplate convertToRecordTemplate(
228221
MetadataChangeProposal mcp, AspectSpec aspectSpec) {
222+
223+
if (mcp.getAspect() == null) {
224+
return null;
225+
}
226+
229227
RecordTemplate aspect;
230228
try {
231229
aspect =

‎metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/DeleteItemImpl.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -94,10 +94,12 @@ public DeleteItemImpl build(AspectRetriever aspectRetriever) {
9494
ValidationApiUtils.validateUrn(aspectRetriever.getEntityRegistry(), this.urn);
9595
log.debug("entity type = {}", this.urn.getEntityType());
9696

97-
entitySpec(aspectRetriever.getEntityRegistry().getEntitySpec(this.urn.getEntityType()));
97+
entitySpec(
98+
ValidationApiUtils.validateEntity(
99+
aspectRetriever.getEntityRegistry(), this.urn.getEntityType()));
98100
log.debug("entity spec = {}", this.entitySpec);
99101

100-
aspectSpec(ValidationApiUtils.validate(this.entitySpec, this.aspectName));
102+
aspectSpec(ValidationApiUtils.validateAspect(this.entitySpec, this.aspectName));
101103
log.debug("aspect spec = {}", this.aspectSpec);
102104

103105
return new DeleteItemImpl(

‎metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCLItemImpl.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,8 @@ public MCLItemImpl build(AspectRetriever aspectRetriever) {
9595
log.debug("entity spec = {}", this.entitySpec);
9696

9797
aspectSpec(
98-
ValidationApiUtils.validate(this.entitySpec, this.metadataChangeLog.getAspectName()));
98+
ValidationApiUtils.validateAspect(
99+
this.entitySpec, this.metadataChangeLog.getAspectName()));
99100
log.debug("aspect spec = {}", this.aspectSpec);
100101

101102
Pair<RecordTemplate, RecordTemplate> aspects =

‎metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/PatchItemImpl.java

+49-33
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import static com.linkedin.metadata.Constants.INGESTION_MAX_SERIALIZED_STRING_LENGTH;
44
import static com.linkedin.metadata.Constants.MAX_JACKSON_STRING_SIZE;
5-
import static com.linkedin.metadata.entity.AspectUtils.validateAspect;
65

76
import com.fasterxml.jackson.core.JsonProcessingException;
87
import com.fasterxml.jackson.core.StreamReadConstraints;
@@ -151,65 +150,82 @@ public ChangeItemImpl applyPatch(RecordTemplate recordTemplate, AspectRetriever
151150

152151
public static class PatchItemImplBuilder {
153152

153+
// Ensure use of other builders
154+
private PatchItemImpl build() {
155+
return null;
156+
}
157+
154158
public PatchItemImpl.PatchItemImplBuilder systemMetadata(SystemMetadata systemMetadata) {
155159
this.systemMetadata = SystemMetadataUtils.generateSystemMetadataIfEmpty(systemMetadata);
156160
return this;
157161
}
158162

163+
public PatchItemImpl.PatchItemImplBuilder aspectSpec(AspectSpec aspectSpec) {
164+
if (!MCPItem.isValidChangeType(ChangeType.PATCH, aspectSpec)) {
165+
throw new UnsupportedOperationException(
166+
"ChangeType not supported: " + ChangeType.PATCH + " for aspect " + this.aspectName);
167+
}
168+
this.aspectSpec = aspectSpec;
169+
return this;
170+
}
171+
172+
public PatchItemImpl.PatchItemImplBuilder patch(JsonPatch patch) {
173+
if (patch == null) {
174+
throw new IllegalArgumentException(String.format("Missing patch to apply. Item: %s", this));
175+
}
176+
this.patch = patch;
177+
return this;
178+
}
179+
159180
public PatchItemImpl build(EntityRegistry entityRegistry) {
160-
ValidationApiUtils.validateUrn(entityRegistry, this.urn);
181+
urn(ValidationApiUtils.validateUrn(entityRegistry, this.urn));
161182
log.debug("entity type = {}", this.urn.getEntityType());
162183

163-
entitySpec(entityRegistry.getEntitySpec(this.urn.getEntityType()));
184+
entitySpec(ValidationApiUtils.validateEntity(entityRegistry, this.urn.getEntityType()));
164185
log.debug("entity spec = {}", this.entitySpec);
165186

166-
aspectSpec(ValidationApiUtils.validate(this.entitySpec, this.aspectName));
187+
aspectSpec(ValidationApiUtils.validateAspect(this.entitySpec, this.aspectName));
167188
log.debug("aspect spec = {}", this.aspectSpec);
168189

169-
if (this.patch == null) {
170-
throw new IllegalArgumentException(
171-
String.format("Missing patch to apply. Aspect: %s", this.aspectSpec.getName()));
190+
if (this.systemMetadata == null) {
191+
// generate default
192+
systemMetadata(null);
172193
}
173194

174195
return new PatchItemImpl(
175196
this.urn,
176197
this.aspectName,
177-
SystemMetadataUtils.generateSystemMetadataIfEmpty(this.systemMetadata),
198+
this.systemMetadata,
178199
this.auditStamp,
179-
this.patch,
200+
Objects.requireNonNull(this.patch),
180201
this.metadataChangeProposal,
181202
this.entitySpec,
182203
this.aspectSpec);
183204
}
184205

185-
public static PatchItemImpl build(
206+
public PatchItemImpl build(
186207
MetadataChangeProposal mcp, AuditStamp auditStamp, EntityRegistry entityRegistry) {
187-
log.debug("entity type = {}", mcp.getEntityType());
188-
EntitySpec entitySpec = entityRegistry.getEntitySpec(mcp.getEntityType());
189-
AspectSpec aspectSpec = validateAspect(mcp, entitySpec);
190208

191-
if (!MCPItem.isValidChangeType(ChangeType.PATCH, aspectSpec)) {
192-
throw new UnsupportedOperationException(
193-
"ChangeType not supported: "
194-
+ mcp.getChangeType()
195-
+ " for aspect "
196-
+ mcp.getAspectName());
197-
}
209+
// Validation includes: Urn, Entity, Aspect
210+
this.metadataChangeProposal = ValidationApiUtils.validateMCP(entityRegistry, mcp);
211+
this.urn = this.metadataChangeProposal.getEntityUrn(); // validation ensures existence
212+
this.auditStamp = auditStamp;
213+
this.aspectName = mcp.getAspectName();
214+
systemMetadata(mcp.getSystemMetadata());
215+
patch(convertToJsonPatch(mcp));
198216

199-
Urn urn = mcp.getEntityUrn();
200-
if (urn == null) {
201-
urn = EntityKeyUtils.getUrnFromProposal(mcp, entitySpec.getKeyAspectSpec());
202-
}
217+
entitySpec(entityRegistry.getEntitySpec(this.urn.getEntityType())); // prior validation
218+
aspectSpec(entitySpec.getAspectSpec(this.aspectName)); // prior validation
203219

204-
return PatchItemImpl.builder()
205-
.urn(urn)
206-
.aspectName(mcp.getAspectName())
207-
.systemMetadata(
208-
SystemMetadataUtils.generateSystemMetadataIfEmpty(mcp.getSystemMetadata()))
209-
.metadataChangeProposal(mcp)
210-
.auditStamp(auditStamp)
211-
.patch(convertToJsonPatch(mcp))
212-
.build(entityRegistry);
220+
return new PatchItemImpl(
221+
this.urn,
222+
this.aspectName,
223+
this.systemMetadata,
224+
this.auditStamp,
225+
this.patch,
226+
this.metadataChangeProposal,
227+
this.entitySpec,
228+
this.aspectSpec);
213229
}
214230

215231
public static JsonPatch convertToJsonPatch(MetadataChangeProposal mcp) {

0 commit comments

Comments
 (0)