Skip to content

Commit 0b80492

Browse files
feat(datahub-client): improve Avro schema conversions
1 parent f9251c8 commit 0b80492

File tree

2 files changed

+195
-104
lines changed

2 files changed

+195
-104
lines changed

metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverter.java

+29-28
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ public class AvroSchemaConverter implements SchemaConverter<Schema> {
2323

2424
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
2525
private static final Map<String, Supplier<SchemaFieldDataType.Type>> LOGICAL_TYPE_MAPPING;
26+
public static final String ARRAY_ITEMS_FIELD_NAME = "items";
27+
public static final String MAP_VALUE_FIELD_NAME = "value";
2628

2729
static {
2830
Map<String, Supplier<SchemaFieldDataType.Type>> logicalTypeMap = new HashMap<>();
@@ -337,6 +339,18 @@ private void processArrayField(
337339
// Set parent type for proper array handling
338340
DataHubType arrayDataHubType = new DataHubType(ArrayType.class, elementType);
339341

342+
SchemaField arrayField =
343+
new SchemaField()
344+
.setFieldPath(fieldPath.asString())
345+
.setType(arrayDataHubType.asSchemaFieldType())
346+
.setNativeDataType("array(" + elementType + ")")
347+
.setNullable(isNullable || defaultNullable)
348+
.setIsPartOfKey(fieldPath.isKeySchema());
349+
350+
populateCommonProperties(field, arrayField);
351+
log.debug("Array field path: {} with doc: {}", fieldPath.asString(), field.doc());
352+
fields.add(arrayField);
353+
340354
// Process element type if it's complex
341355
if (elementSchema.getType() == Schema.Type.RECORD
342356
|| elementSchema.getType() == Schema.Type.ARRAY
@@ -349,25 +363,12 @@ private void processArrayField(
349363
new FieldElement(Collections.singletonList("array"), new ArrayList<>(), null, null));
350364
Schema.Field elementField =
351365
new Schema.Field(
352-
field.name(),
366+
ARRAY_ITEMS_FIELD_NAME,
353367
elementSchema,
354368
elementSchema.getDoc() != null ? elementSchema.getDoc() : field.doc(),
355369
null // TODO: What is the default value for an array element?
356370
);
357-
processField(elementField, fieldPath, defaultNullable, fields, isNullable, arrayDataHubType);
358-
} else {
359-
360-
SchemaField arrayField =
361-
new SchemaField()
362-
.setFieldPath(fieldPath.asString())
363-
.setType(arrayDataHubType.asSchemaFieldType())
364-
.setNativeDataType("array(" + elementType + ")")
365-
.setNullable(isNullable || defaultNullable)
366-
.setIsPartOfKey(fieldPath.isKeySchema());
367-
368-
populateCommonProperties(field, arrayField);
369-
log.debug("Array field path: {} with doc: {}", fieldPath.asString(), field.doc());
370-
fields.add(arrayField);
371+
processField(elementField, fieldPath, defaultNullable, fields, true, arrayDataHubType);
371372
}
372373
}
373374

@@ -386,14 +387,25 @@ private void processMapField(
386387
DataHubType mapDataHubType = new DataHubType(MapType.class, valueType);
387388
fieldPath = fieldPath.expandType("map", mapSchema);
388389

390+
SchemaField mapField =
391+
new SchemaField()
392+
.setFieldPath(fieldPath.asString())
393+
.setType(mapDataHubType.asSchemaFieldType())
394+
.setNativeDataType("map<string," + valueType + ">")
395+
.setNullable(isNullable || defaultNullable)
396+
.setIsPartOfKey(fieldPath.isKeySchema());
397+
398+
populateCommonProperties(field, mapField);
399+
fields.add(mapField);
400+
389401
// Process value type if it's complex
390402
if (valueSchema.getType() == Schema.Type.RECORD
391403
|| valueSchema.getType() == Schema.Type.ARRAY
392404
|| valueSchema.getType() == Schema.Type.MAP
393405
|| valueSchema.getType() == Schema.Type.UNION) {
394406
Schema.Field valueField =
395407
new Schema.Field(
396-
field.name(),
408+
MAP_VALUE_FIELD_NAME,
397409
valueSchema,
398410
valueSchema.getDoc() != null ? valueSchema.getDoc() : field.doc(),
399411
null // TODO: What is the default value for a map value?
@@ -404,18 +416,7 @@ private void processMapField(
404416
.clonePlus(
405417
new FieldElement(
406418
Collections.singletonList("map"), new ArrayList<>(), null, null));
407-
processField(valueField, valueFieldPath, defaultNullable, fields, isNullable, mapDataHubType);
408-
} else {
409-
SchemaField mapField =
410-
new SchemaField()
411-
.setFieldPath(fieldPath.asString())
412-
.setType(mapDataHubType.asSchemaFieldType())
413-
.setNativeDataType("map<string," + valueType + ">")
414-
.setNullable(isNullable || defaultNullable)
415-
.setIsPartOfKey(fieldPath.isKeySchema());
416-
417-
populateCommonProperties(field, mapField);
418-
fields.add(mapField);
419+
processField(valueField, valueFieldPath, defaultNullable, fields, true, mapDataHubType);
419420
}
420421
}
421422

0 commit comments

Comments
 (0)