@@ -596,7 +596,15 @@ class TableauSourceReport(StaleEntityRemovalSourceReport):
596
596
num_datasource_field_skipped_no_name : int = 0
597
597
num_csql_field_skipped_no_name : int = 0
598
598
num_table_field_skipped_no_name : int = 0
599
+ # lineage
600
+ num_tables_with_upstream_lineage : int = 0
601
+ num_upstream_table_lineage : int = 0
602
+ num_upstream_fine_grained_lineage : int = 0
599
603
num_upstream_table_skipped_no_name : int = 0
604
+ num_upstream_table_skipped_no_columns : int = 0
605
+ num_upstream_table_failed_generate_reference : int = 0
606
+ num_upstream_table_lineage_failed_parse_sql : int = 0
607
+ num_upstream_fine_grained_lineage_failed_parse_sql : int = 0
600
608
601
609
602
610
@platform_name ("Tableau" )
@@ -1311,7 +1319,7 @@ def _create_upstream_table_lineage(
1311
1319
datasource : dict ,
1312
1320
browse_path : Optional [str ],
1313
1321
is_embedded_ds : bool = False ,
1314
- ) -> Tuple :
1322
+ ) -> Tuple [ List [ Upstream ], List [ FineGrainedLineage ]] :
1315
1323
upstream_tables : List [Upstream ] = []
1316
1324
fine_grained_lineages : List [FineGrainedLineage ] = []
1317
1325
table_id_to_urn = {}
@@ -1472,6 +1480,7 @@ def get_upstream_tables(
1472
1480
c .COLUMNS_CONNECTION
1473
1481
].get ("totalCount" )
1474
1482
if not is_custom_sql and not num_tbl_cols :
1483
+ self .report .num_upstream_table_skipped_no_columns += 1
1475
1484
logger .warning (
1476
1485
f"Skipping upstream table with id { table [c .ID ]} , no columns: { table } "
1477
1486
)
@@ -1488,6 +1497,7 @@ def get_upstream_tables(
1488
1497
table , default_schema_map = self .config .default_schema_map
1489
1498
)
1490
1499
except Exception as e :
1500
+ self .report .num_upstream_table_failed_generate_reference += 1
1491
1501
self .report .warning (
1492
1502
title = "Potentially Missing Lineage Issue" ,
1493
1503
message = "Failed to generate upstream reference" ,
@@ -1659,15 +1669,7 @@ def get_upstream_fields_from_custom_sql(
1659
1669
func_overridden_info = None , # Here we don't want to override any information from configuration
1660
1670
)
1661
1671
1662
- if parsed_result is None :
1663
- logger .info (
1664
- f"Failed to extract column level lineage from datasource { datasource_urn } "
1665
- )
1666
- return []
1667
- if parsed_result .debug_info .error :
1668
- logger .info (
1669
- f"Failed to extract column level lineage from datasource { datasource_urn } : { parsed_result .debug_info .error } "
1670
- )
1672
+ if parsed_result is None or parsed_result .debug_info .error :
1671
1673
return []
1672
1674
1673
1675
cll : List [ColumnLineageInfo ] = (
@@ -2031,6 +2033,8 @@ def _create_lineage_to_upstream_tables(
2031
2033
aspect_name = c .UPSTREAM_LINEAGE ,
2032
2034
aspect = upstream_lineage ,
2033
2035
)
2036
+ self .report .num_tables_with_upstream_lineage += 1
2037
+ self .report .num_upstream_table_lineage += len (upstream_tables )
2034
2038
2035
2039
@staticmethod
2036
2040
def _clean_tableau_query_parameters (query : str ) -> str :
@@ -2130,7 +2134,7 @@ def parse_custom_sql(
2130
2134
f"Overridden info upstream_db={ upstream_db } , platform_instance={ platform_instance } , platform={ platform } "
2131
2135
)
2132
2136
2133
- return create_lineage_sql_parsed_result (
2137
+ parsed_result = create_lineage_sql_parsed_result (
2134
2138
query = query ,
2135
2139
default_db = upstream_db ,
2136
2140
platform = platform ,
@@ -2140,6 +2144,21 @@ def parse_custom_sql(
2140
2144
schema_aware = not self .config .sql_parsing_disable_schema_awareness ,
2141
2145
)
2142
2146
2147
+ assert parsed_result is not None
2148
+
2149
+ if parsed_result .debug_info .table_error :
2150
+ logger .warning (
2151
+ f"Failed to extract table lineage from datasource { datasource_urn } : { parsed_result .debug_info .table_error } "
2152
+ )
2153
+ self .report .num_upstream_table_lineage_failed_parse_sql += 1
2154
+ elif parsed_result .debug_info .column_error :
2155
+ logger .warning (
2156
+ f"Failed to extract column level lineage from datasource { datasource_urn } : { parsed_result .debug_info .column_error } "
2157
+ )
2158
+ self .report .num_upstream_fine_grained_lineage_failed_parse_sql += 1
2159
+
2160
+ return parsed_result
2161
+
2143
2162
def _enrich_database_tables_with_parsed_schemas (
2144
2163
self , parsing_result : SqlParsingResult
2145
2164
) -> None :
@@ -2174,9 +2193,6 @@ def _create_lineage_from_unsupported_csql(
2174
2193
)
2175
2194
2176
2195
if parsed_result is None :
2177
- logger .info (
2178
- f"Failed to extract table level lineage for datasource { csql_urn } "
2179
- )
2180
2196
return
2181
2197
2182
2198
self ._enrich_database_tables_with_parsed_schemas (parsed_result )
@@ -2196,12 +2212,14 @@ def _create_lineage_from_unsupported_csql(
2196
2212
upstreams = upstream_tables ,
2197
2213
fineGrainedLineages = fine_grained_lineages ,
2198
2214
)
2199
-
2200
2215
yield self .get_metadata_change_proposal (
2201
2216
csql_urn ,
2202
2217
aspect_name = c .UPSTREAM_LINEAGE ,
2203
2218
aspect = upstream_lineage ,
2204
2219
)
2220
+ self .report .num_tables_with_upstream_lineage += 1
2221
+ self .report .num_upstream_table_lineage += len (upstream_tables )
2222
+ self .report .num_upstream_fine_grained_lineage += len (fine_grained_lineages )
2205
2223
2206
2224
def _get_schema_metadata_for_datasource (
2207
2225
self , datasource_fields : List [dict ]
@@ -2352,6 +2370,11 @@ def emit_datasource(
2352
2370
aspect_name = c .UPSTREAM_LINEAGE ,
2353
2371
aspect = upstream_lineage ,
2354
2372
)
2373
+ self .report .num_tables_with_upstream_lineage += 1
2374
+ self .report .num_upstream_table_lineage += len (upstream_tables )
2375
+ self .report .num_upstream_fine_grained_lineage += len (
2376
+ fine_grained_lineages
2377
+ )
2355
2378
2356
2379
# Datasource Fields
2357
2380
schema_metadata = self ._get_schema_metadata_for_datasource (
0 commit comments