From a9c10ac22d851e3bfd7b6f46389fe1622026a874 Mon Sep 17 00:00:00 2001 From: Siddique Bagwan Date: Wed, 4 Dec 2024 11:44:14 +0530 Subject: [PATCH 1/3] remove limit of 100 for pagesize --- .../src/datahub/ingestion/source/tableau/tableau.py | 5 +++++ .../src/datahub/ingestion/source/tableau/tableau_common.py | 2 +- .../tests/integration/tableau/test_tableau_ingest.py | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index 0eafdb4ad23ba..ce2f132364e93 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -377,6 +377,11 @@ class TableauConfig( ) page_size: int = Field( + default=10, + description="[advanced] Nu.", + ) + + number_of_object_in_page: int = Field( default=10, description="[advanced] Number of metadata objects (e.g. CustomSQLTable, PublishedDatasource, etc) to query at a time using the Tableau API.", ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py index ac917c5f128ed..6f7f817f9c47c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py @@ -979,7 +979,7 @@ def get_filter_pages(query_filter: dict, page_size: int) -> List[dict]: len(query_filter.keys()) == 1 and query_filter.get(c.ID_WITH_IN) and isinstance(query_filter[c.ID_WITH_IN], list) - and len(query_filter[c.ID_WITH_IN]) > 100 * page_size + # and len(query_filter[c.ID_WITH_IN]) > 100 * page_size ): ids = query_filter[c.ID_WITH_IN] filter_pages = [ diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py index 6c45b8a47de41..809da1309cdbc 100644 --- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py +++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py @@ -60,7 +60,7 @@ "site": "acryl", "projects": ["default", "Project 2", "Samples"], "extract_project_hierarchy": False, - "page_size": 10, + "page_size": 1000, "ingest_tags": True, "ingest_owner": True, "ingest_tables_external": True, @@ -674,7 +674,7 @@ def test_tableau_ingest_with_platform_instance( "site": "acryl", "platform_instance": "acryl_site1", "projects": ["default", "Project 2"], - "page_size": 10, + "page_size": 1000, "ingest_tags": True, "ingest_owner": True, "ingest_tables_external": True, From 322430cbc13683416d781e220407158f3936ad1d Mon Sep 17 00:00:00 2001 From: Siddique Bagwan Date: Wed, 4 Dec 2024 20:35:30 +0530 Subject: [PATCH 2/3] remove commented line --- .../src/datahub/ingestion/source/tableau/tableau_common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py index 6f7f817f9c47c..c5d14e0afe15a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py @@ -979,7 +979,6 @@ def get_filter_pages(query_filter: dict, page_size: int) -> List[dict]: len(query_filter.keys()) == 1 and query_filter.get(c.ID_WITH_IN) and isinstance(query_filter[c.ID_WITH_IN], list) - # and len(query_filter[c.ID_WITH_IN]) > 100 * page_size ): ids = query_filter[c.ID_WITH_IN] filter_pages = [ From 5653fef9bfe70c36e259df6f71724ff9a6d74143 Mon Sep 17 00:00:00 2001 From: Siddique Bagwan Date: Thu, 5 Dec 2024 20:00:55 +0530 Subject: [PATCH 3/3] address review comments --- .../src/datahub/ingestion/source/tableau/tableau.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index ce2f132364e93..0eafdb4ad23ba 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -377,11 +377,6 @@ class TableauConfig( ) page_size: int = Field( - default=10, - description="[advanced] Nu.", - ) - - number_of_object_in_page: int = Field( default=10, description="[advanced] Number of metadata objects (e.g. CustomSQLTable, PublishedDatasource, etc) to query at a time using the Tableau API.", )