Skip to content

Commit 003f257

Browse files
committed
adding comments for dataset
1 parent 58edf47 commit 003f257

File tree

1 file changed

+8
-0
lines changed
  • metadata-ingestion/src/datahub/ingestion/source

1 file changed

+8
-0
lines changed

metadata-ingestion/src/datahub/ingestion/source/vertexai.py

+8
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,11 @@ def _get_job_output_workunit(self, job: _TrainingJob) -> Iterable[MetadataWorkUn
274274

275275

276276
def _search_dataset(self, dataset_id: str) -> Optional[DatasetProperties]:
277+
"""
278+
Search for a dataset by its ID in Vertex AI.
279+
This method iterates through different types of datasets (Text, Tabular, Image,
280+
TimeSeries, and Video) to find a dataset that matches the given dataset ID.
281+
"""
277282
for ds in self.client.datasets.TextDataset.list():
278283
if ds.name == dataset_id:
279284
return ds
@@ -292,6 +297,9 @@ def _search_dataset(self, dataset_id: str) -> Optional[DatasetProperties]:
292297
return None
293298

294299
def _make_dataset_aspect(self, ds: _Dataset) -> Optional[DatasetPropertiesClass]:
300+
"""
301+
Create a DatasetPropertiesClass aspect for a given Vertex AI dataset.
302+
"""
295303
aspect = DatasetPropertiesClass(
296304
name=self._make_vertexai_name("dataset", ds.name),
297305
created=TimeStampClass(time=int(ds.create_time.timestamp())),

0 commit comments

Comments
 (0)