Skip to content

Commit fda2818

Browse files
committed
refactor(metadata-ingestion): add cachetool to operation_config
1 parent e90cba9 commit fda2818

File tree

3 files changed

+10
-9
lines changed

3 files changed

+10
-9
lines changed

metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from pydantic import Field, validator
77
from pyiceberg.catalog import Catalog, load_catalog
88
from sortedcontainers import SortedList
9+
910
from datahub.configuration.common import AllowDenyPattern, ConfigModel
1011
from datahub.configuration.source_common import DatasetSourceConfigMixin
1112
from datahub.ingestion.source.state.stale_entity_removal_handler import (
@@ -123,7 +124,7 @@ def validate_catalog_size(cls, value):
123124
)
124125

125126
return value
126-
127+
127128
def is_profiling_enabled(self) -> bool:
128129
return self.profiling.enabled and is_profiling_enabled(
129130
self.profiling.operation_config

metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py

+1-4
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22
from abc import abstractmethod
33
from typing import Any, Dict, Optional
44

5-
6-
import cachetools.keys
75
import pydantic
86
from pydantic import Field
97
from sqlalchemy.engine import URL
@@ -30,7 +28,6 @@
3028
)
3129
from datahub.ingestion.source_config.operation_config import is_profiling_enabled
3230

33-
3431
logger: logging.Logger = logging.getLogger(__name__)
3532

3633

@@ -117,7 +114,7 @@ class SQLCommonConfig(
117114
profiling: GEProfilingConfig = GEProfilingConfig()
118115
# Custom Stateful Ingestion settings
119116
stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None
120-
117+
121118
def is_profiling_enabled(self) -> bool:
122119
return self.profiling.enabled and is_profiling_enabled(
123120
self.profiling.operation_config

metadata-ingestion/src/datahub/ingestion/source_config/operation_config.py

+7-4
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
import datetime
22
import logging
33
from typing import Any, Dict, Optional
4+
45
import cachetools
56
import pydantic
67
from pydantic.fields import Field
7-
from datahub.utilities.cachetools_keys import self_methodkey
8+
89
from datahub.configuration.common import ConfigModel
10+
from datahub.utilities.cachetools_keys import self_methodkey
911

1012
logger = logging.getLogger(__name__)
1113

@@ -61,10 +63,11 @@ def validate_profile_date_of_month(cls, v: Optional[int]) -> Optional[int]:
6163
)
6264
return profile_date_of_month
6365

66+
6467
@cachetools.cached(
65-
cache=cachetools.LRUCache(maxsize=1),
66-
key=self_methodkey,
67-
)
68+
cache=cachetools.LRUCache(maxsize=1),
69+
key=self_methodkey,
70+
)
6871
def is_profiling_enabled(operation_config: OperationConfig) -> bool:
6972
if operation_config.lower_freq_profile_enabled is False:
7073
return True

0 commit comments

Comments
 (0)