|
5 | 5 |
|
6 | 6 | from datahub.configuration.pattern_utils import is_schema_allowed
|
7 | 7 | from datahub.emitter.mce_builder import (
|
8 |
| - get_sys_time, |
9 | 8 | make_data_platform_urn,
|
10 | 9 | make_dataset_urn_with_platform_instance,
|
11 | 10 | make_schema_field_urn,
|
|
75 | 74 | PROFILING,
|
76 | 75 | )
|
77 | 76 | from datahub.metadata.com.linkedin.pegasus2avro.common import (
|
78 |
| - AuditStamp, |
79 | 77 | GlobalTags,
|
80 | 78 | Status,
|
81 | 79 | SubTypes,
|
|
102 | 100 | StringType,
|
103 | 101 | TimeType,
|
104 | 102 | )
|
105 |
| -from datahub.metadata.com.linkedin.pegasus2avro.structured import ( |
106 |
| - StructuredPropertyDefinition, |
107 |
| -) |
108 | 103 | from datahub.metadata.com.linkedin.pegasus2avro.tag import TagProperties
|
109 | 104 | from datahub.metadata.urns import (
|
110 |
| - ContainerUrn, |
111 |
| - DatasetUrn, |
112 |
| - DataTypeUrn, |
113 |
| - EntityTypeUrn, |
114 | 105 | SchemaFieldUrn,
|
115 | 106 | StructuredPropertyUrn,
|
116 | 107 | )
|
@@ -705,21 +696,16 @@ def _process_view(
|
705 | 696 | yield from self.gen_dataset_workunits(view, schema_name, db_name)
|
706 | 697 |
|
707 | 698 | def _process_tag(self, tag: SnowflakeTag) -> Iterable[MetadataWorkUnit]:
|
708 |
| - use_sp = self.config.extract_tags_as_structured_properties |
709 |
| - identifier = ( |
710 |
| - self.snowflake_identifier(tag.structured_property_identifier()) |
711 |
| - if use_sp |
712 |
| - else tag.tag_identifier() |
713 |
| - ) |
| 699 | + if self.config.extract_tags_as_structured_properties: |
| 700 | + return |
| 701 | + |
| 702 | + identifier = tag.tag_identifier() |
714 | 703 |
|
715 | 704 | if self.report.is_tag_processed(identifier):
|
716 | 705 | return
|
717 | 706 |
|
718 | 707 | self.report.report_tag_processed(identifier)
|
719 |
| - if use_sp: |
720 |
| - yield from self.gen_tag_as_structured_property_workunits(tag) |
721 |
| - else: |
722 |
| - yield from self.gen_tag_workunits(tag) |
| 708 | + yield from self.gen_tag_workunits(tag) |
723 | 709 |
|
724 | 710 | def _format_tags_as_structured_properties(
|
725 | 711 | self, tags: List[SnowflakeTag]
|
@@ -912,29 +898,6 @@ def gen_tag_workunits(self, tag: SnowflakeTag) -> Iterable[MetadataWorkUnit]:
|
912 | 898 | entityUrn=tag_urn, aspect=tag_properties_aspect
|
913 | 899 | ).as_workunit()
|
914 | 900 |
|
915 |
| - def gen_tag_as_structured_property_workunits( |
916 |
| - self, tag: SnowflakeTag |
917 |
| - ) -> Iterable[MetadataWorkUnit]: |
918 |
| - identifier = self.snowflake_identifier(tag.structured_property_identifier()) |
919 |
| - urn = StructuredPropertyUrn(identifier).urn() |
920 |
| - aspect = StructuredPropertyDefinition( |
921 |
| - qualifiedName=identifier, |
922 |
| - displayName=tag.name, |
923 |
| - valueType=DataTypeUrn("datahub.string").urn(), |
924 |
| - entityTypes=[ |
925 |
| - EntityTypeUrn(f"datahub.{ContainerUrn.ENTITY_TYPE}").urn(), |
926 |
| - EntityTypeUrn(f"datahub.{DatasetUrn.ENTITY_TYPE}").urn(), |
927 |
| - EntityTypeUrn(f"datahub.{SchemaFieldUrn.ENTITY_TYPE}").urn(), |
928 |
| - ], |
929 |
| - lastModified=AuditStamp( |
930 |
| - time=get_sys_time(), actor="urn:li:corpuser:datahub" |
931 |
| - ), |
932 |
| - ) |
933 |
| - yield MetadataChangeProposalWrapper( |
934 |
| - entityUrn=urn, |
935 |
| - aspect=aspect, |
936 |
| - ).as_workunit() |
937 |
| - |
938 | 901 | def gen_column_tags_as_structured_properties(
|
939 | 902 | self, dataset_urn: str, table: Union[SnowflakeTable, SnowflakeView]
|
940 | 903 | ) -> Iterable[MetadataWorkUnit]:
|
|
0 commit comments