Skip to content

Commit 379ed6f

Browse files
PeteMangoPeteMango
committed
restored everything
1 parent 98081cb commit 379ed6f

File tree

2 files changed

+19
-19
lines changed

2 files changed

+19
-19
lines changed

‎metadata-ingestion/src/datahub/ingestion/source/superset.py

+4-8
Original file line numberDiff line numberDiff line change
@@ -401,11 +401,8 @@ def construct_dashboard_from_api_data(
401401
)
402402

403403
modified_actor = f"urn:li:corpuser:{self.owner_info.get((dashboard_data.get('changed_by') or {}).get('id', -1), 'unknown')}"
404-
405-
now = datetime.now().strftime("%I:%M%p on %B %d, %Y")
406-
407404
modified_ts = int(
408-
dp.parse(dashboard_data.get("changed_on", now)).timestamp() * 1000
405+
dp.parse(dashboard_data.get("changed_on_utc", "now")).timestamp() * 1000
409406
)
410407
title = dashboard_data.get("dashboard_title", "")
411408
# note: the API does not currently supply created_by usernames due to a bug
@@ -517,9 +514,8 @@ def construct_chart_from_chart_data(self, chart_data: dict) -> ChartSnapshot:
517514
)
518515

519516
modified_actor = f"urn:li:corpuser:{self.owner_info.get((chart_data.get('changed_by') or {}).get('id', -1), 'unknown')}"
520-
521517
modified_ts = int(
522-
dp.parse(datetime.now().strftime("%I:%M%p on %B %d, %Y")).timestamp() * 1000
518+
dp.parse(chart_data.get("changed_on_utc", "now")).timestamp() * 1000
523519
)
524520
title = chart_data.get("slice_name", "")
525521

@@ -787,10 +783,10 @@ def construct_dataset_from_dataset_data(
787783

788784
modified_actor = f"urn:li:corpuser:{self.owner_info.get((dataset_data.get('changed_by') or {}).get('id', -1), 'unknown')}"
789785

786+
# substitute now in if no timestamp, as modified_ts requires a not None value
790787
now = datetime.now().strftime("%I:%M%p on %B %d, %Y")
791-
792788
modified_ts = int(
793-
dp.parse(dataset_data.get("changed_on", now)).timestamp() * 1000
789+
dp.parse(dataset_data.get("changed_on_utc", now)).timestamp() * 1000
794790
)
795791
last_modified = AuditStampClass(time=modified_ts, actor=modified_actor)
796792

‎metadata-ingestion/tests/integration/superset/test_superset.py

+15-11
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -
4141
"id": 1,
4242
"last_name": "Owners1",
4343
},
44-
"changed_on": "2020-04-14T07:00:00.000000+0000",
44+
"changed_on_utc": "2020-04-14T07:00:00.000000+0000",
4545
"dashboard_title": "test_dashboard_title_1",
4646
"url": "/dashboard/test_dashboard_url_1",
4747
"position_json": '{"CHART-test-1": {"meta": { "chartId": "10" }}, "CHART-test-2": {"meta": { "chartId": "11" }}}',
@@ -69,7 +69,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -
6969
"id": 2,
7070
"last_name": "Owners2",
7171
},
72-
"changed_on": "2020-04-14T07:00:00.000000+0000",
72+
"changed_on_utc": "2020-04-14T07:00:00.000000+0000",
7373
"dashboard_title": "test_dashboard_title_2",
7474
"url": "/dashboard/test_dashboard_url_2",
7575
"position_json": '{"CHART-test-3": {"meta": { "chartId": "12" }}, "CHART-test-4": {"meta": { "chartId": "13" }}}',
@@ -101,6 +101,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -
101101
"id": 1,
102102
"last_name": "Owners1",
103103
},
104+
"changed_on_utc": "2020-04-14T07:00:00.000000+0000",
104105
"slice_name": "test_chart_title_1",
105106
"viz_type": "box_plot",
106107
"url": "/explore/test_chart_url_10",
@@ -114,6 +115,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -
114115
"id": 1,
115116
"last_name": "Owners1",
116117
},
118+
"changed_on_utc": "2020-04-14T07:00:00.000000+0000",
117119
"slice_name": "test_chart_title_2",
118120
"viz_type": "pie",
119121
"url": "/explore/test_chart_url_11",
@@ -127,6 +129,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -
127129
"id": 2,
128130
"last_name": "Owners2",
129131
},
132+
"changed_on_utc": "2020-04-14T07:00:00.000000+0000",
130133
"slice_name": "test_chart_title_3",
131134
"viz_type": "treemap",
132135
"url": "/explore/test_chart_url_12",
@@ -140,6 +143,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -
140143
"id": 2,
141144
"last_name": "Owners2",
142145
},
146+
"changed_on_utc": "2020-04-14T07:00:00.000000+0000",
143147
"slice_name": "test_chart_title_4",
144148
"viz_type": "histogram",
145149
"url": "/explore/test_chart_url_13",
@@ -165,7 +169,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -
165169
},
166170
"changed_by_name": "test_username_1",
167171
"changed_on_delta_humanized": "10 months ago",
168-
"changed_on": "2024-01-05T21:10:15.650819+0000",
172+
"changed_on_utc": "2024-01-05T21:10:15.650819+0000",
169173
"database": {"database_name": "test_database1", "id": 1},
170174
"datasource_type": "table",
171175
"default_endpoint": None,
@@ -192,7 +196,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -
192196
},
193197
"changed_by_name": "test_username_2",
194198
"changed_on_delta_humanized": "9 months ago",
195-
"changed_on": "2024-02-10T15:30:20.123456+0000",
199+
"changed_on_utc": "2024-02-10T15:30:20.123456+0000",
196200
"database": {"database_name": "test_database2", "id": 2},
197201
"datasource_type": "table",
198202
"default_endpoint": None,
@@ -237,7 +241,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -
237241
"verbose_name": "null",
238242
},
239243
],
240-
"changed_on": "2024-02-10T15:30:20.123456+0000",
244+
"changed_on_utc": "2024-02-10T15:30:20.123456+0000",
241245
"database": {"database_name": "test_database1", "id": 1},
242246
"datasource_type": "table",
243247
"default_endpoint": None,
@@ -724,7 +728,7 @@ def test_superset_stateful_ingest(
724728
"id": 1,
725729
"last_name": "Owners1",
726730
},
727-
"changed_on": "2020-04-14T07:00:00.000000+0000",
731+
"changed_on_utc": "2020-04-14T07:00:00.000000+0000",
728732
"dashboard_title": "test_dashboard_title_1",
729733
"url": "/dashboard/test_dashboard_url_1",
730734
"position_json": '{"CHART-test-1": {"meta": { "chartId": "10" }}, "CHART-test-2": {"meta": { "chartId": "11" }}}',
@@ -761,7 +765,7 @@ def test_superset_stateful_ingest(
761765
"id": 1,
762766
"last_name": "Owners1",
763767
},
764-
"changed_on": "2020-04-14T07:00:00.000000+0000",
768+
"changed_on_utc": "2020-04-14T07:00:00.000000+0000",
765769
"slice_name": "test_chart_title_1",
766770
"viz_type": "box_plot",
767771
"url": "/explore/test_chart_url_10",
@@ -775,7 +779,7 @@ def test_superset_stateful_ingest(
775779
"id": 1,
776780
"last_name": "Owners1",
777781
},
778-
"changed_on": "2020-04-14T07:00:00.000000+0000",
782+
"changed_on_utc": "2020-04-14T07:00:00.000000+0000",
779783
"slice_name": "test_chart_title_2",
780784
"viz_type": "pie",
781785
"url": "/explore/test_chart_url_11",
@@ -789,7 +793,7 @@ def test_superset_stateful_ingest(
789793
"id": 2,
790794
"last_name": "Owners2",
791795
},
792-
"changed_on": "2020-04-14T07:00:00.000000+0000",
796+
"changed_on_utc": "2020-04-14T07:00:00.000000+0000",
793797
"slice_name": "test_chart_title_3",
794798
"viz_type": "treemap",
795799
"url": "/explore/test_chart_url_12",
@@ -815,7 +819,7 @@ def test_superset_stateful_ingest(
815819
},
816820
"changed_by_name": "test_username_2",
817821
"changed_on_delta_humanized": "9 months ago",
818-
"changed_on": "2024-02-10T15:30:20.123456+0000",
822+
"changed_on_utc": "2024-02-10T15:30:20.123456+0000",
819823
"database": {"database_name": "test_database1", "id": 1},
820824
"datasource_type": "table",
821825
"default_endpoint": None,
@@ -867,7 +871,7 @@ def test_superset_stateful_ingest(
867871
"verbose_name": "null",
868872
},
869873
],
870-
"changed_on": "2024-02-10T15:30:20.123456+0000",
874+
"changed_on_utc": "2024-02-10T15:30:20.123456+0000",
871875
"database": {"database_name": "test_database1", "id": 1},
872876
"datasource_type": "table",
873877
"default_endpoint": None,

0 commit comments

Comments
 (0)