Skip to content

Commit bc18e1b

Browse files
committed
Update profiler.py
1 parent 18379ea commit bc18e1b

File tree

1 file changed

+8
-0
lines changed
  • metadata-ingestion/src/datahub/ingestion/source/bigquery_v2

1 file changed

+8
-0
lines changed

metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py

+8
Original file line numberDiff line numberDiff line change
@@ -2099,6 +2099,11 @@ def _get_required_partition_filters(
20992099
f"Table {table.name} is an external table, treating as large table for profiling"
21002100
)
21012101
is_small_table = False
2102+
elif table.rows_count == 0 or table.rows_count is None:
2103+
logger.info(
2104+
f"Table {table.name} reports 0 rows or unknown row count, treating more carefully"
2105+
)
2106+
is_small_table = False
21022107
elif (
21032108
table.size_in_bytes is not None
21042109
and table.size_in_bytes < 100_000_000 # Less than 100MB
@@ -2255,6 +2260,9 @@ def get_batch_kwargs(
22552260
or bq_table.rows_count
22562261
and bq_table.rows_count > 50_000_000
22572262
or bq_table.external # IMPORTANT: Always add hints for external tables
2263+
or bq_table.rows_count == 0 # IMPORTANT: Be careful with 0 row tables
2264+
or bq_table.rows_count
2265+
is None # IMPORTANT: Be careful with unknown row counts
22582266
): # > 5GB or external
22592267
needs_optimization_hints = True
22602268

0 commit comments

Comments
 (0)