Skip to content

Commit 3392fac

Browse files
committed
Remove unneded method
lint fix
1 parent 180e3e4 commit 3392fac

File tree

2 files changed

+9
-46
lines changed

2 files changed

+9
-46
lines changed

metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py

+5-42
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55

66
from datahub.configuration.pattern_utils import is_schema_allowed
77
from datahub.emitter.mce_builder import (
8-
get_sys_time,
98
make_data_platform_urn,
109
make_dataset_urn_with_platform_instance,
1110
make_schema_field_urn,
@@ -75,7 +74,6 @@
7574
PROFILING,
7675
)
7776
from datahub.metadata.com.linkedin.pegasus2avro.common import (
78-
AuditStamp,
7977
GlobalTags,
8078
Status,
8179
SubTypes,
@@ -102,15 +100,8 @@
102100
StringType,
103101
TimeType,
104102
)
105-
from datahub.metadata.com.linkedin.pegasus2avro.structured import (
106-
StructuredPropertyDefinition,
107-
)
108103
from datahub.metadata.com.linkedin.pegasus2avro.tag import TagProperties
109104
from datahub.metadata.urns import (
110-
ContainerUrn,
111-
DatasetUrn,
112-
DataTypeUrn,
113-
EntityTypeUrn,
114105
SchemaFieldUrn,
115106
StructuredPropertyUrn,
116107
)
@@ -705,21 +696,16 @@ def _process_view(
705696
yield from self.gen_dataset_workunits(view, schema_name, db_name)
706697

707698
def _process_tag(self, tag: SnowflakeTag) -> Iterable[MetadataWorkUnit]:
708-
use_sp = self.config.extract_tags_as_structured_properties
709-
identifier = (
710-
self.snowflake_identifier(tag.structured_property_identifier())
711-
if use_sp
712-
else tag.tag_identifier()
713-
)
699+
if self.config.extract_tags_as_structured_properties:
700+
return
701+
702+
identifier = tag.tag_identifier()
714703

715704
if self.report.is_tag_processed(identifier):
716705
return
717706

718707
self.report.report_tag_processed(identifier)
719-
if use_sp:
720-
yield from self.gen_tag_as_structured_property_workunits(tag)
721-
else:
722-
yield from self.gen_tag_workunits(tag)
708+
yield from self.gen_tag_workunits(tag)
723709

724710
def _format_tags_as_structured_properties(
725711
self, tags: List[SnowflakeTag]
@@ -912,29 +898,6 @@ def gen_tag_workunits(self, tag: SnowflakeTag) -> Iterable[MetadataWorkUnit]:
912898
entityUrn=tag_urn, aspect=tag_properties_aspect
913899
).as_workunit()
914900

915-
def gen_tag_as_structured_property_workunits(
916-
self, tag: SnowflakeTag
917-
) -> Iterable[MetadataWorkUnit]:
918-
identifier = self.snowflake_identifier(tag.structured_property_identifier())
919-
urn = StructuredPropertyUrn(identifier).urn()
920-
aspect = StructuredPropertyDefinition(
921-
qualifiedName=identifier,
922-
displayName=tag.name,
923-
valueType=DataTypeUrn("datahub.string").urn(),
924-
entityTypes=[
925-
EntityTypeUrn(f"datahub.{ContainerUrn.ENTITY_TYPE}").urn(),
926-
EntityTypeUrn(f"datahub.{DatasetUrn.ENTITY_TYPE}").urn(),
927-
EntityTypeUrn(f"datahub.{SchemaFieldUrn.ENTITY_TYPE}").urn(),
928-
],
929-
lastModified=AuditStamp(
930-
time=get_sys_time(), actor="urn:li:corpuser:datahub"
931-
),
932-
)
933-
yield MetadataChangeProposalWrapper(
934-
entityUrn=urn,
935-
aspect=aspect,
936-
).as_workunit()
937-
938901
def gen_column_tags_as_structured_properties(
939902
self, dataset_urn: str, table: Union[SnowflakeTable, SnowflakeView]
940903
) -> Iterable[MetadataWorkUnit]:

metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_tag.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@
1919
SnowflakeCommonMixin,
2020
SnowflakeIdentifierBuilder,
2121
)
22+
from datahub.metadata.com.linkedin.pegasus2avro.common import AuditStamp
23+
from datahub.metadata.com.linkedin.pegasus2avro.structured import (
24+
StructuredPropertyDefinition,
25+
)
2226
from datahub.metadata.urns import (
2327
ContainerUrn,
2428
DatasetUrn,
@@ -27,10 +31,6 @@
2731
SchemaFieldUrn,
2832
StructuredPropertyUrn,
2933
)
30-
from datahub.metadata.com.linkedin.pegasus2avro.common import AuditStamp
31-
from datahub.metadata.com.linkedin.pegasus2avro.structured import (
32-
StructuredPropertyDefinition,
33-
)
3434

3535
logger: logging.Logger = logging.getLogger(__name__)
3636

0 commit comments

Comments
 (0)