Commit bc18e1b 1 parent 18379ea commit bc18e1b Copy full SHA for bc18e1b
File tree 1 file changed +8
-0
lines changed
metadata-ingestion/src/datahub/ingestion/source/bigquery_v2
1 file changed +8
-0
lines changed Original file line number Diff line number Diff line change @@ -2099,6 +2099,11 @@ def _get_required_partition_filters(
2099
2099
f"Table { table .name } is an external table, treating as large table for profiling"
2100
2100
)
2101
2101
is_small_table = False
2102
+ elif table .rows_count == 0 or table .rows_count is None :
2103
+ logger .info (
2104
+ f"Table { table .name } reports 0 rows or unknown row count, treating more carefully"
2105
+ )
2106
+ is_small_table = False
2102
2107
elif (
2103
2108
table .size_in_bytes is not None
2104
2109
and table .size_in_bytes < 100_000_000 # Less than 100MB
@@ -2255,6 +2260,9 @@ def get_batch_kwargs(
2255
2260
or bq_table .rows_count
2256
2261
and bq_table .rows_count > 50_000_000
2257
2262
or bq_table .external # IMPORTANT: Always add hints for external tables
2263
+ or bq_table .rows_count == 0 # IMPORTANT: Be careful with 0 row tables
2264
+ or bq_table .rows_count
2265
+ is None # IMPORTANT: Be careful with unknown row counts
2258
2266
): # > 5GB or external
2259
2267
needs_optimization_hints = True
2260
2268
You can’t perform that action at this time.
0 commit comments