diff --git a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml index 9b2d767f23bb4..c65a89e4057dd 100644 --- a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml @@ -30,6 +30,9 @@ acceptance_tests: - name: stages - name: deal_products - name: mail + - name: deals + - name: users + - name: persons - config_path: "secrets/config.json" expect_records: path: "integration_tests/expected_records.jsonl" @@ -52,7 +55,33 @@ acceptance_tests: - name: persons incremental: - bypass_reason: "All incremental streams are empty in sandbox account." + tests: + - config_path: secrets/config.json + configured_catalog_path: integration_tests/configured_catalog.json + future_state: + future_state_path: integration_tests/abnormal_state.json + missing_streams: + - name: deals + bypass_reason: "Stream is empty in sandbox account." + - name: files + bypass_reason: "Stream is empty in sandbox account." + - name: filters + bypass_reason: "Stream is empty in sandbox account." + - name: notes + bypass_reason: "Stream is empty in sandbox account." + - name: activities + bypass_reason: "Stream is empty in sandbox account." + - name: persons + bypass_reason: "Stream is empty in sandbox account." + - name: pipelines + bypass_reason: "Stream is empty in sandbox account." + - name: products + bypass_reason: "Stream is empty in sandbox account." + - name: stages + bypass_reason: "Stream is empty in sandbox account." + - name: users + bypass_reason: "Stream is empty in sandbox account." + skip_comprehensive_incremental_tests: true full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-pipedrive/components.py b/airbyte-integrations/connectors/source-pipedrive/components.py index 2f34fa080e105..cf3aa4231e08e 100644 --- a/airbyte-integrations/connectors/source-pipedrive/components.py +++ b/airbyte-integrations/connectors/source-pipedrive/components.py @@ -1,7 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from dataclasses import InitVar, dataclass +from dataclasses import InitVar, dataclass, field from typing import Any, List, Mapping, Union import requests @@ -41,7 +41,7 @@ class NullCheckedDpathExtractor(RecordExtractor): nullable_nested_field: Union[InterpolatedString, str] config: Config parameters: InitVar[Mapping[str, Any]] - decoder: Decoder = JsonDecoder(parameters={}) + decoder: Decoder = field(default_factory=lambda: JsonDecoder(parameters={})) def __post_init__(self, parameters: Mapping[str, Any]): self._dpath_extractor = DpathExtractor( diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/abnormal_state.json index ec68ed9e2a854..dcee6c38d6c02 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/abnormal_state.json @@ -124,7 +124,7 @@ "type": "STREAM", "stream": { "stream_state": { - "update_time": "2217-06-26 21:20:07" + "update_time": "2217-06-26T21:20:07Z" }, "stream_descriptor": { "name": "organizations" diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json index 02422e10fd1c8..f9e741586d654 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json @@ -144,10 +144,13 @@ "stream": { "name": "organizations", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["update_time"] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "cursor_field": ["update_time"], + "destination_sync_mode": "append" }, { "stream": { diff --git a/airbyte-integrations/connectors/source-pipedrive/manifest.yaml b/airbyte-integrations/connectors/source-pipedrive/manifest.yaml index 8bbfd51f148fb..bc6ebee86aead 100644 --- a/airbyte-integrations/connectors/source-pipedrive/manifest.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/manifest.yaml @@ -1,4 +1,4 @@ -version: 6.33.0 +version: 6.41.1 type: DeclarativeSource @@ -21,7 +21,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: deal record_selector: @@ -72,7 +71,6 @@ definitions: path: v1/dealFields http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -105,7 +103,6 @@ definitions: path: v1/goals/find http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -130,7 +127,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: file record_selector: @@ -183,7 +179,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: filter record_selector: @@ -234,7 +229,6 @@ definitions: path: v1/leadLabels http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -256,7 +250,6 @@ definitions: path: v1/leads http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -289,7 +282,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: note record_selector: @@ -342,7 +334,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: activity record_selector: @@ -393,7 +384,6 @@ definitions: path: v1/activityTypes http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -415,7 +405,6 @@ definitions: path: v1/activityFields http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -446,7 +435,6 @@ definitions: path: v1/currencies http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -470,7 +458,6 @@ definitions: path: v1/mailbox/mailThreads/{{ stream_partition.thread_id }}/mailMessages http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -508,12 +495,11 @@ definitions: type: SimpleRetriever requester: $ref: "#/definitions/base_requester" - path: v1/organizations + path: api/v2/organizations http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" - limit: "50" - items: organization + include_fields: >- + ["next_activity_id","last_activity_id","open_deals_count","related_open_deals_count","closed_deals_count","related_closed_deals_count","email_messages_count","people_count","activities_count","done_activities_count","undone_activities_count","files_count","notes_count","followers_count","won_deals_count","related_won_deals_count","lost_deals_count","related_lost_deals_count"] record_selector: type: RecordSelector extractor: @@ -525,10 +511,40 @@ definitions: page_token_option: type: RequestOption inject_into: request_parameter - field_name: start + field_name: cursor + page_size_option: + type: RequestOption + inject_into: request_parameter + field_name: limit pagination_strategy: type: CursorPagination - cursor_value: "{{ response['additional_data']['pagination']['next_start'] }}" + page_size: 100 + cursor_value: "{{ response.get(\"additional_data\", {}).get(\"next_cursor\", {}) }}" + stop_condition: >- + {{ not response.get("additional_data", {}).get("next_cursor", {}) + }} + incremental_sync: + type: DatetimeBasedCursor + cursor_field: update_time + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config[\"replication_start_date\"] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: updated_since + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: updated_until + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" schema_loader: type: InlineSchemaLoader schema: @@ -543,7 +559,6 @@ definitions: path: v1/organizationFields http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -574,7 +589,6 @@ definitions: path: v1/permissionSets http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -598,7 +612,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: person record_selector: @@ -649,7 +662,6 @@ definitions: path: v1/personFields http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -682,7 +694,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: pipeline record_selector: @@ -735,7 +746,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: product record_selector: @@ -786,7 +796,6 @@ definitions: path: v1/productFields http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -817,7 +826,6 @@ definitions: path: v1/roles http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -850,7 +858,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: stage record_selector: @@ -903,7 +910,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: user record_selector: @@ -954,7 +960,6 @@ definitions: path: v1/deals/{{ stream_slice.parent_id }}/products http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -995,7 +1000,6 @@ definitions: path: v1/mailbox/mailThreads http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -1031,6 +1035,13 @@ definitions: base_requester: type: HttpRequester url_base: https://api.pipedrive.com/ + authenticator: + type: ApiKeyAuthenticator + inject_into: + type: RequestOption + inject_into: request_parameter + field_name: api_token + api_token: "{{ config[\"api_token\"] }}" streams: - $ref: "#/definitions/streams/deals" @@ -2805,231 +2816,90 @@ schemas: $schema: http://json-schema.org/draft-07/schema# additionalProperties: true properties: - active_flag: - type: - - "null" - - boolean - activities_count: - type: - - "null" - - number add_time: type: - "null" - string address: - type: - - "null" - - string - address_admin_area_level_1: - type: - - "null" - - string - address_admin_area_level_2: - type: - - "null" - - string - address_country: - type: - - "null" - - string - address_formatted_address: - type: - - "null" - - string - address_locality: - type: - - "null" - - string - address_postal_code: - type: - - "null" - - string - address_route: - type: - - "null" - - string - address_street_number: - type: - - "null" - - string - address_sublocality: - type: - - "null" - - string - address_subpremise: - type: - - "null" - - string - category_id: - type: - - "null" - - integer - cc_email: - type: - - "null" - - string - closed_deals_count: - type: - - "null" - - number - company_id: - type: - - "null" - - number - country_code: - type: - - "null" - - string - delete_time: - type: - - "null" - - string - done_activities_count: - type: - - "null" - - number - email_messages_count: - type: - - "null" - - number - files_count: - type: - - "null" - - number - first_char: - type: - - "null" - - string - followers_count: - type: - - "null" - - number - id: - type: - - "null" - - number - label: - type: - - "null" - - number - last_activity_date: - type: - - "null" - - string - last_activity_id: - type: - - "null" - - number - lost_deals_count: - type: - - "null" - - number - name: - type: - - "null" - - string - next_activity_date: - type: - - "null" - - string - next_activity_id: - type: - - "null" - - number - next_activity_time: - type: - - "null" - - string - notes_count: - type: - - "null" - - number - open_deals_count: - type: - - "null" - - number - owner_id: type: - "null" - object - - number properties: - active_flag: + value: type: - "null" - - boolean - email: + - string + country: type: - "null" - string - has_pic: + admin_area_level_1: type: - "null" - - number - id: + - string + admin_area_level_2: type: - "null" - - number - name: + - string + locality: type: - "null" - string - pic_hash: + sublocality: type: - "null" - string - value: + route: type: - "null" - - number - owner_name: + - string + street_number: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + formatted_address: + type: + - "null" + - string + id: type: - "null" - - string - people_count: + - number + name: type: - "null" - - number - picture_id: + - string + owner_id: type: - "null" - object + - number properties: active_flag: type: - "null" - boolean - add_time: + email: type: - "null" - string - added_by_user_id: + has_pic: type: - "null" - number - item_id: + id: type: - "null" - number - item_type: + name: type: - "null" - string - pictures: - type: - - "null" - - object - properties: - "128": - type: - - "null" - - string - "512": - type: - - "null" - - string - update_time: + pic_hash: type: - "null" - string @@ -3037,26 +2907,22 @@ schemas: type: - "null" - number - related_closed_deals_count: - type: - - "null" - - number - related_lost_deals_count: - type: - - "null" - - number - related_open_deals_count: + org_id: type: - "null" - number - related_won_deals_count: + is_deleted: type: - "null" - - number - undone_activities_count: + - boolean + label_ids: type: - "null" - - number + - array + items: + type: + - "null" + - number update_time: type: - "null" @@ -3066,10 +2932,6 @@ schemas: - "null" - number - string - won_deals_count: - type: - - "null" - - number organization_fields: type: object $schema: http://json-schema.org/draft-07/schema# diff --git a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml index 5076f3bda4fb8..b43a3c5cc148e 100644 --- a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml @@ -17,7 +17,7 @@ data: connectorSubtype: api connectorType: source definitionId: d8286229-c680-4063-8c59-23b9b391c700 - dockerImageTag: 2.4.0 + dockerImageTag: 2.5.0 dockerRepository: airbyte/source-pipedrive documentationUrl: https://docs.airbyte.com/integrations/sources/pipedrive githubIssueLabel: source-pipedrive @@ -63,5 +63,5 @@ data: type: GSM alias: airbyte-connector-testing-secret-store connectorBuildOptions: - baseImage: docker.io/airbyte/source-declarative-manifest:6.33.1@sha256:06468f2b0acdb0126a29757f67025f8f837014f70e3f079e10e304b0e1a6be4b + baseImage: docker.io/airbyte/source-declarative-manifest:6.41.1@sha256:f256b6f008ec07a37e3e6bce3ada17bec4197c1bc66c41c4b756980c8cb587ac metadataSpecVersion: "1.0" diff --git a/docs/integrations/sources/pipedrive.md b/docs/integrations/sources/pipedrive.md index 7268705269db9..7d0626a87f7a5 100644 --- a/docs/integrations/sources/pipedrive.md +++ b/docs/integrations/sources/pipedrive.md @@ -112,6 +112,7 @@ The Pipedrive connector will gracefully handle rate limits. For more information | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.5.0 | 2025-03-14 | [50424](https://github.com/airbytehq/airbyte/pull/50424) | Add incremental sync to organizations stream | | | 2.4.0 | 2025-02-28 | [54716](https://github.com/airbytehq/airbyte/pull/54716) | Refactor: Optimize Parameters, remove redundant code and Improve Manifest Readability | | 2.3.8 | 2025-02-22 | [47292](https://github.com/airbytehq/airbyte/pull/47292) | Migrate to manifest only format | | 2.3.7 | 2025-02-08 | [53488](https://github.com/airbytehq/airbyte/pull/53488) | Update dependencies |