From 98a8c5c52c21c5fdcb207ab219d2db0c4d1f6aac Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Thu, 6 Mar 2025 13:33:13 -0800 Subject: [PATCH 1/3] fix(Amazon Seller Partner): Fix daterange in `DatetimeBasedCursor` and Added configurable step size for financial events streams --- .../metadata.yaml | 2 +- .../pyproject.toml | 2 +- .../manifest.yaml | 88 ++++++++++--------- .../source_amazon_seller_partner/spec.json | 75 +++++++++++++--- .../sources/amazon-seller-partner.md | 25 ++++-- 5 files changed, 128 insertions(+), 64 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml index ec6ecdfea3462..1344ed7769e8c 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml @@ -15,7 +15,7 @@ data: connectorSubtype: api connectorType: source definitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460 - dockerImageTag: 4.6.0 + dockerImageTag: 4.6.1 dockerRepository: airbyte/source-amazon-seller-partner documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-seller-partner erdUrl: https://dbdocs.io/airbyteio/source-amazon-seller-partner?view=relationships diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/pyproject.toml b/airbyte-integrations/connectors/source-amazon-seller-partner/pyproject.toml index dc697cfbea5e0..fe70faf6332c2 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/pyproject.toml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/pyproject.toml @@ -3,7 +3,7 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "4.6.0" +version = "4.6.1" name = "source-amazon-seller-partner" description = "Source implementation for Amazon Seller Partner." authors = ["Airbyte "] diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/manifest.yaml b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/manifest.yaml index c277b5f69502e..5938facd5209e 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/manifest.yaml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/manifest.yaml @@ -43,7 +43,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', (now_utc() - duration('PT2M')).strftime('%Y-%m-%dT%H:%M:%SZ') ), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else (now_utc() - duration('PT2M')).strftime('%Y-%m-%dT%H:%M:%SZ'), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: field_name: LastUpdatedAfter @@ -113,7 +113,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" transformations: - type: AddFields @@ -195,7 +195,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: field_name: createdAfter @@ -269,7 +269,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: field_name: changedAfter @@ -340,7 +340,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc() - duration('PT5M') ), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc() - duration('PT5M'), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: field_name: FinancialEventGroupStartedAfter @@ -348,6 +348,8 @@ definitions: end_time_option: field_name: FinancialEventGroupStartedBefore inject_into: request_parameter + step: "{{ config['financial_events_step'] }}" + cursor_granularity: "PT1S" retriever: type: SimpleRetriever requester: @@ -408,7 +410,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc() - duration('PT5M') ), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc() - duration('PT5M'), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: field_name: PostedAfter @@ -416,6 +418,8 @@ definitions: end_time_option: field_name: PostedBefore inject_into: request_parameter + step: "{{ config['financial_events_step'] }}" + cursor_granularity: "PT1S" transformations: - type: AddFields fields: @@ -477,7 +481,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -508,7 +512,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -540,7 +544,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -578,7 +582,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -616,7 +620,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -653,7 +657,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -691,7 +695,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -729,7 +733,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -770,7 +774,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -816,7 +820,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -862,7 +866,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -900,7 +904,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -938,7 +942,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -987,7 +991,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1025,7 +1029,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1063,7 +1067,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1101,7 +1105,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1138,7 +1142,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1176,7 +1180,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1214,7 +1218,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1252,7 +1256,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1298,7 +1302,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1336,7 +1340,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1382,7 +1386,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1420,7 +1424,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1458,7 +1462,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1496,7 +1500,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1534,7 +1538,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1572,7 +1576,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1610,7 +1614,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1648,7 +1652,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1688,7 +1692,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1730,7 +1734,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -1776,7 +1780,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" lookback_window: "P1D" retriever: @@ -1823,7 +1827,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: $ref: "#/definitions/basic_async_retriever" @@ -2042,7 +2046,7 @@ definitions: inject_into: "request_parameter" end_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(config.get('replication_end_date', now_utc()), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(config['replication_end_date'] if config.get('replication_end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_time_option: field_name: "createdUntil" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json index 486ff8b82115e..63f2a69ca6010 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json @@ -25,7 +25,10 @@ "aws_environment": { "title": "AWS Environment", "description": "Select the AWS Environment.", - "enum": ["PRODUCTION", "SANDBOX"], + "enum": [ + "PRODUCTION", + "SANDBOX" + ], "default": "PRODUCTION", "type": "string", "order": 1 @@ -64,7 +67,10 @@ "account_type": { "title": "AWS Seller Partner Account Type", "description": "Type of the Account you're going to authorize the Airbyte application by", - "enum": ["Seller", "Vendor"], + "enum": [ + "Seller", + "Vendor" + ], "default": "Seller", "type": "string", "order": 3 @@ -101,7 +107,9 @@ "title": "Start Date", "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. If start date is not provided or older than 2 years ago from today, the date 2 years ago from today will be used.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"], + "examples": [ + "2017-01-25T00:00:00Z" + ], "order": 8, "type": "string", "format": "date-time" @@ -110,7 +118,9 @@ "title": "End Date", "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data after this date will not be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$|^$", - "examples": ["2017-01-25T00:00:00Z"], + "examples": [ + "2017-01-25T00:00:00Z" + ], "order": 9, "type": "string", "format": "date-time" @@ -131,7 +141,11 @@ "items": { "type": "object", "title": "Report Options", - "required": ["report_name", "stream_name", "options_list"], + "required": [ + "report_name", + "stream_name", + "options_list" + ], "properties": { "report_name": { "title": "Report Name", @@ -189,7 +203,10 @@ "order": 2, "items": { "type": "object", - "required": ["option_name", "option_value"], + "required": [ + "option_name", + "option_value" + ], "properties": { "option_name": { "title": "Name", @@ -213,12 +230,30 @@ "description": "For report based streams with known amount of requests per time period, this option will use waiting time between requests to avoid fatal statuses in reports. See Troubleshooting section for more details", "default": false, "order": 12 + }, + "financial_events_step": { + "title": "Financial Events Step Size", + "description": "The time window size for fetching financial events data in chunks. Options are 1 day (P1D), 7 days (P7D), or 14 days (P14D), based on API limitations.\n\n- **Smaller step sizes (e.g., 1 day)** are better for large data volumes. They fetch smaller chunks per request, reducing the risk of timeouts or overwhelming the API, though more requests may slow syncing and increase the chance of hitting rate limits.\n- **Larger step sizes (e.g., 14 days)** are better for smaller data volumes. They fetch more data per request, speeding up syncing and reducing the number of API calls, which minimizes strain on rate limits.\n\nSelect a step size that matches your data volume to optimize syncing speed and API performance.", + "type": "string", + "enum": [ + "P1D", + "P7D", + "P14D", + "P30D", + "P60D", + "P90D", + "P180D" + ], + "default": "P14D", + "order": 13 } } }, "advanced_auth": { "auth_flow_type": "oauth2.0", - "predicate_key": ["auth_type"], + "predicate_key": [ + "auth_type" + ], "predicate_value": "oauth2.0", "oauth_config_specification": { "oauth_user_input_from_connector_config_specification": { @@ -226,11 +261,15 @@ "properties": { "region": { "type": "string", - "path_in_connector_config": ["region"] + "path_in_connector_config": [ + "region" + ] }, "account_type": { "type": "string", - "path_in_connector_config": ["account_type"] + "path_in_connector_config": [ + "account_type" + ] } } }, @@ -240,7 +279,9 @@ "properties": { "refresh_token": { "type": "string", - "path_in_connector_config": ["refresh_token"] + "path_in_connector_config": [ + "refresh_token" + ] } } }, @@ -265,18 +306,24 @@ "properties": { "app_id": { "type": "string", - "path_in_connector_config": ["app_id"] + "path_in_connector_config": [ + "app_id" + ] }, "lwa_app_id": { "type": "string", - "path_in_connector_config": ["lwa_app_id"] + "path_in_connector_config": [ + "lwa_app_id" + ] }, "lwa_client_secret": { "type": "string", - "path_in_connector_config": ["lwa_client_secret"] + "path_in_connector_config": [ + "lwa_client_secret" + ] } } } } } -} +} \ No newline at end of file diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md index f70acfe828687..2a3be39dc9f97 100644 --- a/docs/integrations/sources/amazon-seller-partner.md +++ b/docs/integrations/sources/amazon-seller-partner.md @@ -64,9 +64,13 @@ To pass the check for Seller and Vendor accounts, you must have access to the [O 6. Log in and Authorize to your Amazon Seller Partner account. 7. For `Start Date`, enter the date in `YYYY-MM-DD` format. The data added on and after this date will be replicated. This field is optional - if not provided or older than 2 years ago from today, the date 2 years ago from today will be used. 8. For `End Date`, enter the date in `YYYY-MM-DD` format. Any data after this date will not be replicated. This field is optional - if not provided, today's date will be used. -9. You can specify report options for each stream using **Report Options** section. Available options can be found in corresponding category [here](https://developer-docs.amazon.com/sp-api/docs/report-type-values). -10. For `Wait between requests to avoid fatal statuses in reports`, enable if you want to use wating time between requests to avoid fatal statuses in report based streams. -11. Click `Set up source`. +9. **Financial Events Step Size**: Select the time window size for fetching financial events data. Options include: + - `P1D` (1 day) + - `P7D` (7 days) + - `P14D` (14 days, default) +10. You can specify report options for each stream using **Report Options** section. Available options can be found in corresponding category [here](https://developer-docs.amazon.com/sp-api/docs/report-type-values). +11. For `Wait between requests to avoid fatal statuses in reports`, enable if you want to use wating time between requests to avoid fatal statuses in report based streams. +12. Click `Set up source`. @@ -80,9 +84,13 @@ To pass the check for Seller and Vendor accounts, you must have access to the [O 4. Using developer application from Step 1, [generate](https://developer-docs.amazon.com/sp-api/docs/self-authorization) refresh token. 5. For Start Date, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. This field is optional - if not provided, the date 2 years ago from today will be used. 6. For End Date, enter the date in YYYY-MM-DD format. Any data after this date will not be replicated. This field is optional - if not provided, today's date will be used. -7. You can specify report options for each stream using **Report Options** section. Available options can be found in corresponding category [here](https://developer-docs.amazon.com/sp-api/docs/report-type-values). -8. For `Wait between requests to avoid fatal statuses in reports`, enable if you want to use wating time between requests to avoid fatal statuses in report based streams. -9. Click `Set up source`. +7. **Financial Events Step Size**: Select the time window size for fetching financial events data. Options include: + - `P1D` (1 day) + - `P7D` (7 days) + - `P14D` (14 days, default) +8. You can specify report options for each stream using **Report Options** section. Available options can be found in corresponding category [here](https://developer-docs.amazon.com/sp-api/docs/report-type-values). +9. For `Wait between requests to avoid fatal statuses in reports`, enable if you want to use wating time between requests to avoid fatal statuses in report based streams. +10. Click `Set up source`. @@ -172,6 +180,10 @@ but with different options for the `sellingProgram` parameter - `FRESH` and `RET Information about rate limits you may find [here](https://developer-docs.amazon.com/sp-api/docs/usage-plans-and-rate-limits-in-the-sp-api). +- Use the **Financial Events Step Size** configuration: + - **Smaller step sizes** (e.g., `P1D` or 1 day) are ideal for large data volumes to avoid timeouts and reduce the risk of hitting rate limits. + - **Larger step sizes** (e.g., `P14D` or 14 days) are more efficient for smaller data volumes, reducing the number of API calls. + ## Data type map | Integration Type | Airbyte Type | @@ -228,6 +240,7 @@ Create a separate connection for streams which usually fail with error above "Fa | Version | Date | Pull Request | Subject | |:-----------|:-----------|:----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 4.6.1 | 2024-10-01 | [TBD](https://github.com/airbytehq/airbyte/pull/TBD) |Fix daterange in `DatetimeBasedCursor` and Added configurable step size for financial events streams (`list_financial_event_groups`, `list_financial_events`)| | 4.6.0 | 2025-02-24 | [53225](https://github.com/airbytehq/airbyte/pull/53225) | Add API Budget | | 4.5.3 | 2025-02-22 | [53928](https://github.com/airbytehq/airbyte/pull/53928) | Update dependencies | | 4.5.2 | 2025-02-17 | [53693](https://github.com/airbytehq/airbyte/pull/53693) | Add app_id to server configuration (OAuth) | From 754ed5f417055a93bfdd06cb00f47679855bbdff Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Thu, 6 Mar 2025 21:38:52 +0000 Subject: [PATCH 2/3] chore: auto-fix lint and format issues --- .../source_amazon_seller_partner/spec.json | 69 ++++--------------- 1 file changed, 15 insertions(+), 54 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json index 63f2a69ca6010..258b093203368 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json @@ -25,10 +25,7 @@ "aws_environment": { "title": "AWS Environment", "description": "Select the AWS Environment.", - "enum": [ - "PRODUCTION", - "SANDBOX" - ], + "enum": ["PRODUCTION", "SANDBOX"], "default": "PRODUCTION", "type": "string", "order": 1 @@ -67,10 +64,7 @@ "account_type": { "title": "AWS Seller Partner Account Type", "description": "Type of the Account you're going to authorize the Airbyte application by", - "enum": [ - "Seller", - "Vendor" - ], + "enum": ["Seller", "Vendor"], "default": "Seller", "type": "string", "order": 3 @@ -107,9 +101,7 @@ "title": "Start Date", "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. If start date is not provided or older than 2 years ago from today, the date 2 years ago from today will be used.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": [ - "2017-01-25T00:00:00Z" - ], + "examples": ["2017-01-25T00:00:00Z"], "order": 8, "type": "string", "format": "date-time" @@ -118,9 +110,7 @@ "title": "End Date", "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data after this date will not be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$|^$", - "examples": [ - "2017-01-25T00:00:00Z" - ], + "examples": ["2017-01-25T00:00:00Z"], "order": 9, "type": "string", "format": "date-time" @@ -141,11 +131,7 @@ "items": { "type": "object", "title": "Report Options", - "required": [ - "report_name", - "stream_name", - "options_list" - ], + "required": ["report_name", "stream_name", "options_list"], "properties": { "report_name": { "title": "Report Name", @@ -203,10 +189,7 @@ "order": 2, "items": { "type": "object", - "required": [ - "option_name", - "option_value" - ], + "required": ["option_name", "option_value"], "properties": { "option_name": { "title": "Name", @@ -235,15 +218,7 @@ "title": "Financial Events Step Size", "description": "The time window size for fetching financial events data in chunks. Options are 1 day (P1D), 7 days (P7D), or 14 days (P14D), based on API limitations.\n\n- **Smaller step sizes (e.g., 1 day)** are better for large data volumes. They fetch smaller chunks per request, reducing the risk of timeouts or overwhelming the API, though more requests may slow syncing and increase the chance of hitting rate limits.\n- **Larger step sizes (e.g., 14 days)** are better for smaller data volumes. They fetch more data per request, speeding up syncing and reducing the number of API calls, which minimizes strain on rate limits.\n\nSelect a step size that matches your data volume to optimize syncing speed and API performance.", "type": "string", - "enum": [ - "P1D", - "P7D", - "P14D", - "P30D", - "P60D", - "P90D", - "P180D" - ], + "enum": ["P1D", "P7D", "P14D", "P30D", "P60D", "P90D", "P180D"], "default": "P14D", "order": 13 } @@ -251,9 +226,7 @@ }, "advanced_auth": { "auth_flow_type": "oauth2.0", - "predicate_key": [ - "auth_type" - ], + "predicate_key": ["auth_type"], "predicate_value": "oauth2.0", "oauth_config_specification": { "oauth_user_input_from_connector_config_specification": { @@ -261,15 +234,11 @@ "properties": { "region": { "type": "string", - "path_in_connector_config": [ - "region" - ] + "path_in_connector_config": ["region"] }, "account_type": { "type": "string", - "path_in_connector_config": [ - "account_type" - ] + "path_in_connector_config": ["account_type"] } } }, @@ -279,9 +248,7 @@ "properties": { "refresh_token": { "type": "string", - "path_in_connector_config": [ - "refresh_token" - ] + "path_in_connector_config": ["refresh_token"] } } }, @@ -306,24 +273,18 @@ "properties": { "app_id": { "type": "string", - "path_in_connector_config": [ - "app_id" - ] + "path_in_connector_config": ["app_id"] }, "lwa_app_id": { "type": "string", - "path_in_connector_config": [ - "lwa_app_id" - ] + "path_in_connector_config": ["lwa_app_id"] }, "lwa_client_secret": { "type": "string", - "path_in_connector_config": [ - "lwa_client_secret" - ] + "path_in_connector_config": ["lwa_client_secret"] } } } } } -} \ No newline at end of file +} From 0f7b93a2c3b9afbc26ad0e107ebcd0d99ece54c3 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Thu, 6 Mar 2025 13:47:09 -0800 Subject: [PATCH 3/3] update doc --- docs/integrations/sources/amazon-seller-partner.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md index 2a3be39dc9f97..0214b163c1d81 100644 --- a/docs/integrations/sources/amazon-seller-partner.md +++ b/docs/integrations/sources/amazon-seller-partner.md @@ -240,7 +240,7 @@ Create a separate connection for streams which usually fail with error above "Fa | Version | Date | Pull Request | Subject | |:-----------|:-----------|:----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 4.6.1 | 2024-10-01 | [TBD](https://github.com/airbytehq/airbyte/pull/TBD) |Fix daterange in `DatetimeBasedCursor` and Added configurable step size for financial events streams (`list_financial_event_groups`, `list_financial_events`)| +| 4.6.1 | 2024-10-01 | [55238](https://github.com/airbytehq/airbyte/pull/55238) |Fix daterange in `DatetimeBasedCursor` and Added configurable step size for financial events streams (`list_financial_event_groups`, `list_financial_events`)| | 4.6.0 | 2025-02-24 | [53225](https://github.com/airbytehq/airbyte/pull/53225) | Add API Budget | | 4.5.3 | 2025-02-22 | [53928](https://github.com/airbytehq/airbyte/pull/53928) | Update dependencies | | 4.5.2 | 2025-02-17 | [53693](https://github.com/airbytehq/airbyte/pull/53693) | Add app_id to server configuration (OAuth) |