feat(ingest): reporting logging tweaks (#9835)

Author: hsheth2

metadata-ingestion/src/datahub/cli/check_cli.py

@@ -218,5 +218,7 @@ def sql_lineage(
     )
 
     logger.debug("Sql parsing debug info: %s", lineage.debug_info)
+    if lineage.debug_info.error:
+        logger.debug("Sql parsing error details", exc_info=lineage.debug_info.error)
 
     click.echo(lineage.json(indent=4))

metadata-ingestion/src/datahub/ingestion/api/source.py

@@ -1,4 +1,5 @@
 import datetime
+import logging
 from abc import ABCMeta, abstractmethod
 from collections import defaultdict
 from dataclasses import dataclass, field
@@ -40,6 +41,8 @@
 from datahub.utilities.lossy_collections import LossyDict, LossyList
 from datahub.utilities.type_annotations import get_class_from_annotation
 
+logger = logging.getLogger(__name__)
+
 
 class SourceCapability(Enum):
     PLATFORM_INSTANCE = "Platform Instance"
@@ -99,11 +102,19 @@ def report_warning(self, key: str, reason: str) -> None:
         warnings.append(reason)
         self.warnings[key] = warnings
 
+    def warning(self, key: str, reason: str) -> None:
+        self.report_warning(key, reason)
+        logger.warning(f"{key} => {reason}", stacklevel=2)
+
     def report_failure(self, key: str, reason: str) -> None:
         failures = self.failures.get(key, LossyList())
         failures.append(reason)
         self.failures[key] = failures
 
+    def failure(self, key: str, reason: str) -> None:
+        self.report_failure(key, reason)
+        logger.error(f"{key} => {reason}", stacklevel=2)
+
     def __post_init__(self) -> None:
         self.start_time = datetime.datetime.now()
         self.running_time: datetime.timedelta = datetime.timedelta(seconds=0)

metadata-ingestion/src/datahub/ingestion/source_report/ingestion_stage.py

@@ -30,7 +30,8 @@ def report_ingestion_stage_start(self, stage: str) -> None:
         if self._timer:
             elapsed = round(self._timer.elapsed_seconds(), 2)
             logger.info(
-                f"Time spent in stage <{self.ingestion_stage}>: {elapsed} seconds"
+                f"Time spent in stage <{self.ingestion_stage}>: {elapsed} seconds",
+                stacklevel=2,
             )
             if self.ingestion_stage:
                 self.ingestion_stage_durations[self.ingestion_stage] = elapsed

metadata-ingestion/src/datahub/utilities/logging_manager.py

@@ -23,7 +23,7 @@
 from datahub.utilities.tee_io import TeeIO
 
 BASE_LOGGING_FORMAT = (
-    "[%(asctime)s] %(levelname)-8s {%(name)s:%(lineno)d} - %(message)s"
+    "[%(asctime)s] %(levelname)-8s {%(filename)s:%(lineno)d} - %(message)s"
 )
 DATAHUB_PACKAGES = [
     "datahub",