From 36c3a245ad3f58cc026ef22f602aaa689e92681e Mon Sep 17 00:00:00 2001 From: kyleromines <163910384+kyleromines@users.noreply.github.com> Date: Mon, 17 Mar 2025 17:24:33 -0400 Subject: [PATCH 1/4] Remove custom incremental class --- .../connectors/source-outreach/components.py | 14 -- .../connectors/source-outreach/manifest.yaml | 175 ++++++++++++------ 2 files changed, 122 insertions(+), 67 deletions(-) diff --git a/airbyte-integrations/connectors/source-outreach/components.py b/airbyte-integrations/connectors/source-outreach/components.py index f5cd821725346..253ae5fcf250d 100644 --- a/airbyte-integrations/connectors/source-outreach/components.py +++ b/airbyte-integrations/connectors/source-outreach/components.py @@ -36,17 +36,3 @@ def extract_records(self, response: requests.Response, **kwargs) -> List[Mapping return extracted_records - -@dataclass -class CustomIncrementalSync(DatetimeBasedCursor): - def get_request_params( - self, - *, - stream_state: Optional[StreamState] = None, - stream_slice: Optional[StreamSlice] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> MutableMapping[str, Any]: - params = {} - if self.cursor_field.string in stream_state: - params[f"filter[{self.cursor_field.string}]"] = stream_state[self.cursor_field.string] + "..inf" - return params diff --git a/airbyte-integrations/connectors/source-outreach/manifest.yaml b/airbyte-integrations/connectors/source-outreach/manifest.yaml index 43f0f78512718..11b59064e701e 100644 --- a/airbyte-integrations/connectors/source-outreach/manifest.yaml +++ b/airbyte-integrations/connectors/source-outreach/manifest.yaml @@ -24,6 +24,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -38,15 +39,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -66,6 +70,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -80,15 +85,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -108,6 +116,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -122,15 +131,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -150,6 +162,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -164,15 +177,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -192,6 +208,7 @@ definitions: count: "false" sort: updatedAt page[size]: "100" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -206,15 +223,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -234,6 +254,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -248,15 +269,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -276,6 +300,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -290,15 +315,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -318,6 +346,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -332,15 +361,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -360,6 +392,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -374,15 +407,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -402,6 +438,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -416,15 +453,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -444,6 +484,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -458,15 +499,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -486,6 +530,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -500,15 +545,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -528,6 +576,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -542,15 +591,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -570,6 +622,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -584,15 +637,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -612,6 +668,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -626,15 +683,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -654,6 +714,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -668,15 +729,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -696,6 +760,7 @@ definitions: count: "false" sort: updatedAt page[size]: "1000" + newFilterSyntax: "true" record_selector: type: RecordSelector extractor: @@ -710,15 +775,18 @@ definitions: cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}" stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}" incremental_sync: - type: CustomIncrementalSync - class_name: source_declarative_manifest.components.CustomIncrementalSync - datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%_msZ" cursor_field: updatedAt is_data_feed: true start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: "filter[updatedAt][gte]" + inject_into: "request_parameter" schema_loader: type: InlineSchemaLoader schema: @@ -794,11 +862,12 @@ spec: type: string description: >- The date from which you'd like to replicate data for Outreach API, in - the format YYYY-MM-DDT00:00:00Z. All data generated after this date + the format YYYY-MM-DDT00:00:00.000Z. All data generated after this date will be replicated. title: Start Date + format: date-time examples: - - "2020-11-16T00:00:00Z" + - "2020-11-16T00:00:00.000Z" pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ order: 4 additionalProperties: true From 200580a6b005c91545f7ad772449f3e8aeee6f6d Mon Sep 17 00:00:00 2001 From: kyleromines <163910384+kyleromines@users.noreply.github.com> Date: Mon, 17 Mar 2025 17:38:24 -0400 Subject: [PATCH 2/4] Update outreach.md --- docs/integrations/sources/outreach.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/sources/outreach.md b/docs/integrations/sources/outreach.md index 05766bf6f6781..e33cd749bab63 100644 --- a/docs/integrations/sources/outreach.md +++ b/docs/integrations/sources/outreach.md @@ -56,6 +56,7 @@ List of available streams: | Version | Date | Pull Request | Subject | | :------ |:-----------| :----- | :------ | +| 1.1.7 | 2025-03-17 | [55805](https://github.com/airbytehq/airbyte/pull/55805/files) | Remove custom incremental class and use new filter syntax | | 1.1.6 | 2025-03-09 | [55180](https://github.com/airbytehq/airbyte/pull/55180) | 🐛 Source Outreach: remove stream_state interpolation | | 1.1.5 | 2025-03-08 | [55561](https://github.com/airbytehq/airbyte/pull/55561) | Update dependencies | | 1.1.4 | 2025-03-01 | [54611](https://github.com/airbytehq/airbyte/pull/54611) | Update dependencies | From cff5fce24e25d1cd232c48ff4286976c8364c5cb Mon Sep 17 00:00:00 2001 From: kyleromines <163910384+kyleromines@users.noreply.github.com> Date: Mon, 17 Mar 2025 17:41:56 -0400 Subject: [PATCH 3/4] Update manifest.yaml --- airbyte-integrations/connectors/source-outreach/manifest.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-outreach/manifest.yaml b/airbyte-integrations/connectors/source-outreach/manifest.yaml index 11b59064e701e..cea1f1c8068d2 100644 --- a/airbyte-integrations/connectors/source-outreach/manifest.yaml +++ b/airbyte-integrations/connectors/source-outreach/manifest.yaml @@ -862,12 +862,12 @@ spec: type: string description: >- The date from which you'd like to replicate data for Outreach API, in - the format YYYY-MM-DDT00:00:00.000Z. All data generated after this date + the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated. title: Start Date format: date-time examples: - - "2020-11-16T00:00:00.000Z" + - "2020-11-16T00:00:00Z" pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ order: 4 additionalProperties: true From d94534cd11532a38463843e3f3bc91d9ac56afe6 Mon Sep 17 00:00:00 2001 From: kyleromines <163910384+kyleromines@users.noreply.github.com> Date: Tue, 18 Mar 2025 09:19:40 -0400 Subject: [PATCH 4/4] ruff --- airbyte-integrations/connectors/source-outreach/components.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-outreach/components.py b/airbyte-integrations/connectors/source-outreach/components.py index 253ae5fcf250d..1388013d02d73 100644 --- a/airbyte-integrations/connectors/source-outreach/components.py +++ b/airbyte-integrations/connectors/source-outreach/components.py @@ -35,4 +35,3 @@ def extract_records(self, response: requests.Response, **kwargs) -> List[Mapping extracted_records.append(extracted_record) return extracted_records -