Skip to content

Commit 3f7d7dd

Browse files
PeteMangoPeteMango
committed
physical dataset lineage fixed
1 parent 9d3291d commit 3f7d7dd

File tree

2 files changed

+33
-35
lines changed

2 files changed

+33
-35
lines changed

‎metadata-ingestion/src/datahub/ingestion/source/superset.py

-2
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,6 @@
8080
OwnerClass,
8181
OwnershipClass,
8282
OwnershipTypeClass,
83-
SchemaFieldDataTypeClass,
8483
TagAssociationClass,
8584
UpstreamClass,
8685
UpstreamLineageClass,
@@ -755,7 +754,6 @@ def generate_physical_dataset_lineage(
755754
downstream=DownstreamColumnRef(
756755
table=None,
757756
column=column.get("column_name", ""),
758-
column_type=SchemaFieldDataTypeClass(column.get("type"), ""),
759757
native_column_type=column.get("type", ""),
760758
),
761759
upstreams=[

‎metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json

+33-33
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@
6464
},
6565
"systemMetadata": {
6666
"lastObserved": 1586847600000,
67-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
67+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
6868
"lastRunId": "no-run-id-provided",
6969
"pipelineName": "test_pipeline"
7070
}
@@ -122,7 +122,7 @@
122122
},
123123
"systemMetadata": {
124124
"lastObserved": 1586847600000,
125-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
125+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
126126
"lastRunId": "no-run-id-provided",
127127
"pipelineName": "test_pipeline"
128128
}
@@ -180,7 +180,7 @@
180180
},
181181
"systemMetadata": {
182182
"lastObserved": 1586847600000,
183-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
183+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
184184
"lastRunId": "no-run-id-provided",
185185
"pipelineName": "test_pipeline"
186186
}
@@ -238,7 +238,7 @@
238238
},
239239
"systemMetadata": {
240240
"lastObserved": 1586847600000,
241-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
241+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
242242
"lastRunId": "no-run-id-provided",
243243
"pipelineName": "test_pipeline"
244244
}
@@ -379,7 +379,7 @@
379379
},
380380
"systemMetadata": {
381381
"lastObserved": 1586847600000,
382-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
382+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
383383
"lastRunId": "no-run-id-provided",
384384
"pipelineName": "test_pipeline"
385385
}
@@ -517,7 +517,7 @@
517517
},
518518
"systemMetadata": {
519519
"lastObserved": 1586847600000,
520-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
520+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
521521
"lastRunId": "no-run-id-provided",
522522
"pipelineName": "test_pipeline"
523523
}
@@ -658,7 +658,7 @@
658658
},
659659
"systemMetadata": {
660660
"lastObserved": 1586847600000,
661-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
661+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
662662
"lastRunId": "no-run-id-provided",
663663
"pipelineName": "test_pipeline"
664664
}
@@ -796,7 +796,7 @@
796796
},
797797
"systemMetadata": {
798798
"lastObserved": 1586847600000,
799-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
799+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
800800
"lastRunId": "no-run-id-provided",
801801
"pipelineName": "test_pipeline"
802802
}
@@ -937,7 +937,7 @@
937937
},
938938
"systemMetadata": {
939939
"lastObserved": 1586847600000,
940-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
940+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
941941
"lastRunId": "no-run-id-provided",
942942
"pipelineName": "test_pipeline"
943943
}
@@ -1075,7 +1075,7 @@
10751075
},
10761076
"systemMetadata": {
10771077
"lastObserved": 1586847600000,
1078-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
1078+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
10791079
"lastRunId": "no-run-id-provided",
10801080
"pipelineName": "test_pipeline"
10811081
}
@@ -1216,7 +1216,7 @@
12161216
},
12171217
"systemMetadata": {
12181218
"lastObserved": 1586847600000,
1219-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
1219+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
12201220
"lastRunId": "no-run-id-provided",
12211221
"pipelineName": "test_pipeline"
12221222
}
@@ -1354,7 +1354,7 @@
13541354
},
13551355
"systemMetadata": {
13561356
"lastObserved": 1586847600000,
1357-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
1357+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
13581358
"lastRunId": "no-run-id-provided",
13591359
"pipelineName": "test_pipeline"
13601360
}
@@ -1495,7 +1495,7 @@
14951495
},
14961496
"systemMetadata": {
14971497
"lastObserved": 1586847600000,
1498-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
1498+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
14991499
"lastRunId": "no-run-id-provided",
15001500
"pipelineName": "test_pipeline"
15011501
}
@@ -1633,7 +1633,7 @@
16331633
},
16341634
"systemMetadata": {
16351635
"lastObserved": 1586847600000,
1636-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
1636+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
16371637
"lastRunId": "no-run-id-provided",
16381638
"pipelineName": "test_pipeline"
16391639
}
@@ -1774,7 +1774,7 @@
17741774
},
17751775
"systemMetadata": {
17761776
"lastObserved": 1586847600000,
1777-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
1777+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
17781778
"lastRunId": "no-run-id-provided",
17791779
"pipelineName": "test_pipeline"
17801780
}
@@ -1912,7 +1912,7 @@
19121912
},
19131913
"systemMetadata": {
19141914
"lastObserved": 1586847600000,
1915-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
1915+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
19161916
"lastRunId": "no-run-id-provided",
19171917
"pipelineName": "test_pipeline"
19181918
}
@@ -2053,7 +2053,7 @@
20532053
},
20542054
"systemMetadata": {
20552055
"lastObserved": 1586847600000,
2056-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
2056+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
20572057
"lastRunId": "no-run-id-provided",
20582058
"pipelineName": "test_pipeline"
20592059
}
@@ -2191,7 +2191,7 @@
21912191
},
21922192
"systemMetadata": {
21932193
"lastObserved": 1586847600000,
2194-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
2194+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
21952195
"lastRunId": "no-run-id-provided",
21962196
"pipelineName": "test_pipeline"
21972197
}
@@ -2332,7 +2332,7 @@
23322332
},
23332333
"systemMetadata": {
23342334
"lastObserved": 1586847600000,
2335-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
2335+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
23362336
"lastRunId": "no-run-id-provided",
23372337
"pipelineName": "test_pipeline"
23382338
}
@@ -2470,7 +2470,7 @@
24702470
},
24712471
"systemMetadata": {
24722472
"lastObserved": 1586847600000,
2473-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
2473+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
24742474
"lastRunId": "no-run-id-provided",
24752475
"pipelineName": "test_pipeline"
24762476
}
@@ -2611,7 +2611,7 @@
26112611
},
26122612
"systemMetadata": {
26132613
"lastObserved": 1586847600000,
2614-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
2614+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
26152615
"lastRunId": "no-run-id-provided",
26162616
"pipelineName": "test_pipeline"
26172617
}
@@ -2749,7 +2749,7 @@
27492749
},
27502750
"systemMetadata": {
27512751
"lastObserved": 1586847600000,
2752-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
2752+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
27532753
"lastRunId": "no-run-id-provided",
27542754
"pipelineName": "test_pipeline"
27552755
}
@@ -2766,7 +2766,7 @@
27662766
},
27672767
"systemMetadata": {
27682768
"lastObserved": 1586847600000,
2769-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
2769+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
27702770
"lastRunId": "no-run-id-provided",
27712771
"pipelineName": "test_pipeline"
27722772
}
@@ -2783,7 +2783,7 @@
27832783
},
27842784
"systemMetadata": {
27852785
"lastObserved": 1586847600000,
2786-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
2786+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
27872787
"lastRunId": "no-run-id-provided",
27882788
"pipelineName": "test_pipeline"
27892789
}
@@ -2800,7 +2800,7 @@
28002800
},
28012801
"systemMetadata": {
28022802
"lastObserved": 1586847600000,
2803-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
2803+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
28042804
"lastRunId": "no-run-id-provided",
28052805
"pipelineName": "test_pipeline"
28062806
}
@@ -2817,14 +2817,14 @@
28172817
},
28182818
"systemMetadata": {
28192819
"lastObserved": 1586847600000,
2820-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
2820+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
28212821
"lastRunId": "no-run-id-provided",
28222822
"pipelineName": "test_pipeline"
28232823
}
28242824
},
28252825
{
2826-
"entityType": "chart",
2827-
"entityUrn": "urn:li:chart:(superset,13)",
2826+
"entityType": "dashboard",
2827+
"entityUrn": "urn:li:dashboard:(superset,2)",
28282828
"changeType": "UPSERT",
28292829
"aspectName": "status",
28302830
"aspect": {
@@ -2834,7 +2834,7 @@
28342834
},
28352835
"systemMetadata": {
28362836
"lastObserved": 1586847600000,
2837-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
2837+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
28382838
"lastRunId": "no-run-id-provided",
28392839
"pipelineName": "test_pipeline"
28402840
}
@@ -2851,14 +2851,14 @@
28512851
},
28522852
"systemMetadata": {
28532853
"lastObserved": 1586847600000,
2854-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
2854+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
28552855
"lastRunId": "no-run-id-provided",
28562856
"pipelineName": "test_pipeline"
28572857
}
28582858
},
28592859
{
2860-
"entityType": "dashboard",
2861-
"entityUrn": "urn:li:dashboard:(superset,2)",
2860+
"entityType": "chart",
2861+
"entityUrn": "urn:li:chart:(superset,13)",
28622862
"changeType": "UPSERT",
28632863
"aspectName": "status",
28642864
"aspect": {
@@ -2868,7 +2868,7 @@
28682868
},
28692869
"systemMetadata": {
28702870
"lastObserved": 1586847600000,
2871-
"runId": "superset-2020_04_14-07_00_00-0sx9yn",
2871+
"runId": "superset-2020_04_14-07_00_00-0r0zcs",
28722872
"lastRunId": "no-run-id-provided",
28732873
"pipelineName": "test_pipeline"
28742874
}

0 commit comments

Comments
 (0)