chore(ruff): enable some ignored rules (#12815)

Author: sgomezvillamor

metadata-ingestion/pyproject.toml

@@ -43,19 +43,13 @@ extend-ignore = [
     "RUF015", # unnecessary-iterable-allocation-for-first-element
 
     # Can be enabled later if someone wants to fix existing cases or an auto-fix becomes available
-    "SIM101", # Multiple isinstance calls for {name}, merge into a single call
     "SIM102", # Use a single `if` statement instead of nested `if` statements
     "SIM103", # Return the condition directly
     "SIM105", # Use `contextlib.suppress(...)` instead of `try`-`except`-`pass`
-    "SIM108", # Use ternary operator {contents} instead of if-else-block
-    "SIM110", # Use `return any(re.match(regex_pattern, path, re.IGNORECASE) for path in paths)` instead of `for` loop
-    "SIM113", # Use enumerate() for index variable {index} in for loop
     "SIM115", # Use a context manager for opening files
     "SIM116", # Use a dictionary instead of consecutive `if` statements
     "SIM117", # Use a single with statement with multiple contexts instead of nested with statements
     "SIM118", # Use key {operator} dict instead of key {operator} dict.keys()
-    "SIM210", # Use `bool(...)` instead of `True if ... else False`
-    "SIM401", # Use `sample_data.get(schema_field.fieldPath, [])` instead of an `if` block
 ]
 
 [tool.ruff.lint.mccabe]

metadata-ingestion/src/datahub/configuration/common.py

@@ -130,7 +130,7 @@ class PermissiveConfigModel(ConfigModel):
     # It is usually used for argument bags that are passed through to third-party libraries.
 
     class Config:
-        if PYDANTIC_VERSION_2:
+        if PYDANTIC_VERSION_2:  # noqa: SIM108
             extra = "allow"
         else:
             extra = Extra.allow

metadata-ingestion/src/datahub/ingestion/glossary/classification_mixin.py

@@ -279,11 +279,7 @@ def get_columns_to_classify(
                             "Dataset_Name": dataset_name,
                         }
                     ),
-                    values=(
-                        sample_data[schema_field.fieldPath]
-                        if schema_field.fieldPath in sample_data
-                        else []
-                    ),
+                    values=sample_data.get(schema_field.fieldPath, []),
                 )
             )
 

metadata-ingestion/src/datahub/ingestion/graph/client.py

@@ -782,9 +782,7 @@ def get_container_urns_by_filter(
         results: Dict = self._post_generic(url, search_body)
         num_entities = results["value"]["numEntities"]
         logger.debug(f"Matched {num_entities} containers")
-        entities_yielded: int = 0
         for x in results["value"]["entities"]:
-            entities_yielded += 1
             logger.debug(f"yielding {x['entity']}")
             yield x["entity"]
 

metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py

@@ -163,7 +163,7 @@ def _convert_sets_to_lists(obj: Any) -> Any:
                 key: DatahubIngestionRunSummaryProvider._convert_sets_to_lists(value)
                 for key, value in obj.items()
             }
-        elif isinstance(obj, list) or isinstance(obj, set):
+        elif isinstance(obj, (list, set)):
             return [
                 DatahubIngestionRunSummaryProvider._convert_sets_to_lists(element)
                 for element in obj

metadata-ingestion/src/datahub/ingestion/run/pipeline.py

@@ -526,10 +526,8 @@ def process_commits(self) -> None:
         Evaluates the commit_policy for each committable in the context and triggers the commit operation
         on the committable if its required commit policies are satisfied.
         """
-        has_errors: bool = (
-            True
-            if self.source.get_report().failures or self.sink.get_report().failures
-            else False
+        has_errors: bool = bool(
+            self.source.get_report().failures or self.sink.get_report().failures
         )
         has_warnings: bool = bool(
             self.source.get_report().warnings or self.sink.get_report().warnings

metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py

@@ -454,10 +454,8 @@ def get_partition_from_path(self, path: str) -> Optional[List[Tuple[str, str]]]:
                     return None
                 partition = partition_split[0]
                 # If partition is in the form of /value1/value2/value3 we infer it from the path and assign partition_0, partition_1, partition_2 etc
-                num = 0
-                for partition_value in partition.split("/"):
+                for num, partition_value in enumerate(partition.split("/")):
                     partition_keys.append((f"partition_{num}", partition_value))
-                    num += 1
             return partition_keys
 
         return None

metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py

@@ -1774,10 +1774,8 @@ def _aggregate_owners(
                     logger.debug(
                         f"Owner after applying owner extraction pattern:'{self.config.owner_extraction_pattern}' is '{owner}'."
                     )
-            if isinstance(owner, list):
-                owners = owner
-            else:
-                owners = [owner]
+            owners = owner if isinstance(owner, list) else [owner]
+
             for owner in owners:
                 if self.config.strip_user_ids_from_email:
                     owner = owner.split("@")[0]

metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_tests.py

@@ -57,15 +57,11 @@ def _get_name_for_relationship_test(kw_args: Dict[str, str]) -> Optional[str]:
         # base assertions are violated, bail early
         return None
     m = re.match(r"^ref\(\'(.*)\'\)$", destination_ref)
-    if m:
-        destination_table = m.group(1)
-    else:
-        destination_table = destination_ref
+    destination_table = m.group(1) if m else destination_ref
+
     m = re.search(r"ref\(\'(.*)\'\)", source_ref)
-    if m:
-        source_table = m.group(1)
-    else:
-        source_table = source_ref
+    source_table = m.group(1) if m else source_ref
+
     return f"{source_table}.{column_name} referential integrity to {destination_table}.{dest_field_name}"
 
 

metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py

@@ -683,11 +683,7 @@ def _check_pattern_match(
                 # Add end anchor for exact matching
                 regex_pattern = regex_pattern + "$"
 
-        for path in paths:
-            if re.match(regex_pattern, path, re.IGNORECASE):
-                return True
-
-        return False
+        return any(re.match(regex_pattern, path, re.IGNORECASE) for path in paths)
 
     def should_include_container(self, path: List[str], name: str) -> bool:
         """

metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_aspects.py

@@ -116,10 +116,7 @@ def get_field_type(
             data_type = data_type.lower()
             type_class = cls.FIELD_TYPE_MAPPING.get(data_type, NullTypeClass)
 
-            if data_size:
-                native_data_type = f"{data_type}({data_size})"
-            else:
-                native_data_type = data_type
+            native_data_type = f"{data_type}({data_size})" if data_size else data_type
 
         try:
             schema_field_type = SchemaFieldDataTypeClass(type=type_class())

metadata-ingestion/src/datahub/ingestion/source/kafka_connect/common.py

@@ -141,12 +141,7 @@ def get_dataset_name(
     database_name: Optional[str],
     source_table: str,
 ) -> str:
-    if database_name:
-        dataset_name = database_name + "." + source_table
-    else:
-        dataset_name = source_table
-
-    return dataset_name
+    return database_name + "." + source_table if database_name else source_table
 
 
 def get_platform_instance(

metadata-ingestion/src/datahub/ingestion/source/sql/athena.py

@@ -540,19 +540,13 @@ def get_schema_fields_for_column(
             inspector=inspector,
             description=column.get("comment"),
             nullable=column.get("nullable", True),
-            is_part_of_key=(
-                True
-                if (
-                    pk_constraints is not None
-                    and isinstance(pk_constraints, dict)
-                    and column["name"] in pk_constraints.get("constrained_columns", [])
-                )
-                else False
+            is_part_of_key=bool(
+                pk_constraints is not None
+                and isinstance(pk_constraints, dict)
+                and column["name"] in pk_constraints.get("constrained_columns", [])
             ),
-            is_partitioning_key=(
-                True
-                if (partition_keys is not None and column["name"] in partition_keys)
-                else False
+            is_partitioning_key=bool(
+                partition_keys is not None and column["name"] in partition_keys
             ),
         )
 

metadata-ingestion/src/datahub/ingestion/source/sql/hive.py

@@ -821,12 +821,8 @@ def _process_view(
 
         try:
             view_definition = inspector.get_view_definition(view, schema)
-            if view_definition is None:
-                view_definition = ""
-            else:
-                # Some dialects return a TextClause instead of a raw string,
-                # so we need to convert them to a string.
-                view_definition = str(view_definition)
+            # Some dialects return a TextClause instead of a raw string, so we need to convert them to a string.
+            view_definition = str(view_definition) if view_definition else ""
         except NotImplementedError:
             view_definition = ""
 

metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py

@@ -893,8 +893,9 @@ def get_schema_fields_for_column(
         return get_schema_fields_for_hive_column(
             column["col_name"],
             column["col_type"],
+            # column is actually an sqlalchemy.engine.row.LegacyRow, not a Dict and we cannot make column.get("col_description", "")
             description=(
-                column["col_description"] if "col_description" in column else ""
+                column["col_description"] if "col_description" in column else ""  # noqa: SIM401
             ),
             default_nullable=True,
         )

metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py

@@ -1031,16 +1031,10 @@ def loop_views(
     def _get_view_definition(self, inspector: Inspector, schema: str, view: str) -> str:
         try:
             view_definition = inspector.get_view_definition(view, schema)
-            if view_definition is None:
-                view_definition = ""
-            else:
-                # Some dialects return a TextClause instead of a raw string,
-                # so we need to convert them to a string.
-                view_definition = str(view_definition)
+            # Some dialects return a TextClause instead of a raw string, so we need to convert them to a string.
+            return str(view_definition) if view_definition else ""
         except NotImplementedError:
-            view_definition = ""
-
-        return view_definition
+            return ""
 
     def _process_view(
         self,

metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py

@@ -114,14 +114,10 @@ def __init__(
         self.stateful_ingestion_config: Optional[StatefulStaleMetadataRemovalConfig] = (
             config.stateful_ingestion
         )
-        self.checkpointing_enabled: bool = (
-            True
-            if (
-                self.state_provider.is_stateful_ingestion_configured()
-                and self.stateful_ingestion_config
-                and self.stateful_ingestion_config.remove_stale_metadata
-            )
-            else False
+        self.checkpointing_enabled: bool = bool(
+            self.state_provider.is_stateful_ingestion_configured()
+            and self.stateful_ingestion_config
+            and self.stateful_ingestion_config.remove_stale_metadata
         )
         self._job_id = self._init_job_id()
         self._urns_to_skip: Set[str] = set()

metadata-ingestion/src/datahub/ingestion/source/superset.py

@@ -431,9 +431,7 @@ def construct_dashboard_from_api_data(
                     dashboard_data.get("owners", []),
                 )
             ),
-            "IsCertified": str(
-                True if dashboard_data.get("certified_by") else False
-            ).lower(),
+            "IsCertified": str(bool(dashboard_data.get("certified_by"))).lower(),
         }
 
         if dashboard_data.get("certified_by"):

metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py

@@ -902,7 +902,7 @@ def get_unique_custom_sql(custom_sql_list: List[dict]) -> List[dict]:
             "name": custom_sql.get("name"),
             # We assume that this is unsupported custom sql if "actual tables that this query references"
             # are missing from api result.
-            "isUnsupportedCustomSql": True if not custom_sql.get("tables") else False,
+            "isUnsupportedCustomSql": not custom_sql.get("tables"),
             "query": custom_sql.get("query"),
             "connectionType": custom_sql.get("connectionType"),
             "columns": custom_sql.get("columns"),

metadata-ingestion/src/datahub/lite/duckdb_lite.py

@@ -760,9 +760,7 @@ def post_update_hook(
                 entity_id=[str(data_platform_urn), data_platform_instance],
             )
             self._create_edges_from_data_platform_instance(data_platform_instance_urn)
-        elif isinstance(aspect, ChartInfoClass) or isinstance(
-            aspect, DashboardInfoClass
-        ):
+        elif isinstance(aspect, (ChartInfoClass, DashboardInfoClass)):
             urn = Urn.from_string(entity_urn)
             self.add_edge(
                 entity_urn,

metadata-ingestion/src/datahub/sdk/dataset.py

@@ -74,8 +74,8 @@
 def _parse_upstream_input(
     upstream_input: UpstreamInputType,
 ) -> Union[models.UpstreamClass, models.FineGrainedLineageClass]:
-    if isinstance(upstream_input, models.UpstreamClass) or isinstance(
-        upstream_input, models.FineGrainedLineageClass
+    if isinstance(
+        upstream_input, (models.UpstreamClass, models.FineGrainedLineageClass)
     ):
         return upstream_input
     elif isinstance(upstream_input, (str, DatasetUrn)):

metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py

@@ -56,10 +56,7 @@ def get_dialect(platform: DialectOrStr) -> sqlglot.Dialect:
 def is_dialect_instance(
     dialect: sqlglot.Dialect, platforms: Union[str, Iterable[str]]
 ) -> bool:
-    if isinstance(platforms, str):
-        platforms = [platforms]
-    else:
-        platforms = list(platforms)
+    platforms = [platforms] if isinstance(platforms, str) else list(platforms)
 
     dialects = [get_dialect(platform) for platform in platforms]
 

metadata-ingestion/tests/performance/bigquery/bigquery_events.py

@@ -99,7 +99,7 @@ def generate_events(
                     if config.debug_include_full_payloads
                     else None
                 ),
-                query_on_view=True if referencedViews else False,
+                query_on_view=bool(referencedViews),
             )
         )
         table_accesses: Dict[BigQueryTableRef, Set[str]] = defaultdict(set)