From 08505b8a74d6aa6398a9f8812b0def0afff10aa6 Mon Sep 17 00:00:00 2001 From: Tope Folorunso Date: Thu, 26 Dec 2024 02:39:30 +0100 Subject: [PATCH 01/11] add incremental sync to organizations stream --- .../integration_tests/configured_catalog.json | 9 ++++++--- .../connectors/source-pipedrive/metadata.yaml | 2 +- .../source_pipedrive/manifest.yaml | 16 +++++++++++++++- docs/integrations/sources/pipedrive.md | 1 + 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json index 02422e10fd1c8..f9e741586d654 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json @@ -144,10 +144,13 @@ "stream": { "name": "organizations", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["update_time"] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "cursor_field": ["update_time"], + "destination_sync_mode": "append" }, { "stream": { diff --git a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml index 6233ea2e546ed..5af11b0899486 100644 --- a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml @@ -17,7 +17,7 @@ data: connectorSubtype: api connectorType: source definitionId: d8286229-c680-4063-8c59-23b9b391c700 - dockerImageTag: 2.3.0 + dockerImageTag: 2.4.0 dockerRepository: airbyte/source-pipedrive documentationUrl: https://docs.airbyte.com/integrations/sources/pipedrive githubIssueLabel: source-pipedrive diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml index a49dd6bd90643..a6bbcda47b0bc 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml @@ -232,9 +232,23 @@ definitions: primary_key: "id" retriever: $ref: "#/definitions/retriever" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: update_time + cursor_datetime_formats: + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%d %H:%M:%S" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['replication_start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: since_timestamp $parameters: path_extractor: "organization" - path: "/organizations" + path: "/recents" organization_fields_stream: $ref: "#/definitions/base_stream" diff --git a/docs/integrations/sources/pipedrive.md b/docs/integrations/sources/pipedrive.md index a64cb0d8ead87..45bf24b2ac76a 100644 --- a/docs/integrations/sources/pipedrive.md +++ b/docs/integrations/sources/pipedrive.md @@ -112,6 +112,7 @@ The Pipedrive connector will gracefully handle rate limits. For more information | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.4.0 | 2024-12-26 | [x](https://github.com/airbytehq/airbyte/pull/x) | Add incremental sync to organizations stream | | 2.3.0 | 2024-12-17 | [48615](https://github.com/airbytehq/airbyte/pull/48615) | Update airbyte-cdk to use concurrency | | 2.2.28 | 2024-12-14 | [49692](https://github.com/airbytehq/airbyte/pull/49692) | Update dependencies | | 2.2.27 | 2024-12-12 | [49041](https://github.com/airbytehq/airbyte/pull/49041) | Starting with this version, the Docker image is now rootless. Please note that this and future versions will not be compatible with Airbyte versions earlier than 0.64 | From 0293211567e06ff371be19af5db9755554909e45 Mon Sep 17 00:00:00 2001 From: Tope Folorunso <66448986+topefolorunso@users.noreply.github.com> Date: Thu, 26 Dec 2024 08:41:05 +0100 Subject: [PATCH 02/11] Update pipedrive.md --- docs/integrations/sources/pipedrive.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/pipedrive.md b/docs/integrations/sources/pipedrive.md index 45bf24b2ac76a..9ce0a99234d6e 100644 --- a/docs/integrations/sources/pipedrive.md +++ b/docs/integrations/sources/pipedrive.md @@ -112,7 +112,7 @@ The Pipedrive connector will gracefully handle rate limits. For more information | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 2.4.0 | 2024-12-26 | [x](https://github.com/airbytehq/airbyte/pull/x) | Add incremental sync to organizations stream | +| 2.4.0 | 2024-12-26 | [50424](https://github.com/airbytehq/airbyte/pull/50424) | Add incremental sync to organizations stream | | 2.3.0 | 2024-12-17 | [48615](https://github.com/airbytehq/airbyte/pull/48615) | Update airbyte-cdk to use concurrency | | 2.2.28 | 2024-12-14 | [49692](https://github.com/airbytehq/airbyte/pull/49692) | Update dependencies | | 2.2.27 | 2024-12-12 | [49041](https://github.com/airbytehq/airbyte/pull/49041) | Starting with this version, the Docker image is now rootless. Please note that this and future versions will not be compatible with Airbyte versions earlier than 0.64 | From 90b2871ad3765ddf60493a5c6b0ded9e0b3a1dca Mon Sep 17 00:00:00 2001 From: Tope Folorunso <66448986+topefolorunso@users.noreply.github.com> Date: Thu, 26 Dec 2024 08:41:56 +0100 Subject: [PATCH 03/11] Update pyproject.toml --- airbyte-integrations/connectors/source-pipedrive/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-pipedrive/pyproject.toml b/airbyte-integrations/connectors/source-pipedrive/pyproject.toml index 588cb5d6444db..7ec0ea062247f 100644 --- a/airbyte-integrations/connectors/source-pipedrive/pyproject.toml +++ b/airbyte-integrations/connectors/source-pipedrive/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "2.3.0" +version = "2.4.0" name = "source-pipedrive" description = "Source implementation for Pipedrive." authors = [ "Airbyte ",] From 06ac338c07f5e7b5015223ab708ebf1acd67e354 Mon Sep 17 00:00:00 2001 From: Tope Folorunso Date: Fri, 14 Mar 2025 14:55:23 +0100 Subject: [PATCH 04/11] fix --- .../source-railz/unit_tests/pyproject.toml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/source-railz/unit_tests/pyproject.toml b/airbyte-integrations/connectors/source-railz/unit_tests/pyproject.toml index 4c5858f386030..689f5ba608fef 100644 --- a/airbyte-integrations/connectors/source-railz/unit_tests/pyproject.toml +++ b/airbyte-integrations/connectors/source-railz/unit_tests/pyproject.toml @@ -14,19 +14,10 @@ homepage = "https://airbyte.com" repository = "https://github.com/airbytehq/airbyte" [tool.poetry.dependencies] -python = "^3.9,<3.12" -airbyte-cdk = "^0" -isodate = "==0.6.1" -requests = "==2.28.2" - -[tool.poetry.scripts] -source-railz = "source_railz.run:run" - -[tool.poetry.group.dev.dependencies] +python = "^3.10,<3.13" +airbyte-cdk = "6.31.1" +pytest = "^8" pytest-mock = "^3.6.1" -requests-mock = "^1.9.3" -pytest = "^6.2" -freezegun = "^1.5.1" [tool.pytest.ini_options] filterwarnings = [ From 3113bd4741de457b92d481fde37ee06cec405edb Mon Sep 17 00:00:00 2001 From: Tope Folorunso Date: Fri, 14 Mar 2025 16:11:43 +0100 Subject: [PATCH 05/11] fix --- airbyte-integrations/connectors/source-pipedrive/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml index 5076f3bda4fb8..6ce046d1d9d48 100644 --- a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml @@ -17,7 +17,7 @@ data: connectorSubtype: api connectorType: source definitionId: d8286229-c680-4063-8c59-23b9b391c700 - dockerImageTag: 2.4.0 + dockerImageTag: 2.5.0 dockerRepository: airbyte/source-pipedrive documentationUrl: https://docs.airbyte.com/integrations/sources/pipedrive githubIssueLabel: source-pipedrive From 2058e8089964bb9f9b02220a2a92caf7251ee78d Mon Sep 17 00:00:00 2001 From: Tope Folorunso Date: Sat, 15 Mar 2025 19:42:51 +0100 Subject: [PATCH 06/11] fix path --- .../connectors/source-pipedrive/manifest.yaml | 69 +++++++++---------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/airbyte-integrations/connectors/source-pipedrive/manifest.yaml b/airbyte-integrations/connectors/source-pipedrive/manifest.yaml index 656525188f378..c6fd2a2af5d20 100644 --- a/airbyte-integrations/connectors/source-pipedrive/manifest.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/manifest.yaml @@ -1,4 +1,4 @@ -version: 6.33.0 +version: 6.39.3 type: DeclarativeSource @@ -21,7 +21,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: deal record_selector: @@ -72,7 +71,6 @@ definitions: path: v1/dealFields http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -105,7 +103,6 @@ definitions: path: v1/goals/find http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -130,7 +127,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: file record_selector: @@ -183,7 +179,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: filter record_selector: @@ -234,7 +229,6 @@ definitions: path: v1/leadLabels http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -256,7 +250,6 @@ definitions: path: v1/leads http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -289,7 +282,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: note record_selector: @@ -342,7 +334,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: activity record_selector: @@ -393,7 +384,6 @@ definitions: path: v1/activityTypes http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -415,7 +405,6 @@ definitions: path: v1/activityFields http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -446,7 +435,6 @@ definitions: path: v1/currencies http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -470,7 +458,6 @@ definitions: path: v1/mailbox/mailThreads/{{ stream_partition.thread_id }}/mailMessages http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -508,12 +495,11 @@ definitions: type: SimpleRetriever requester: $ref: "#/definitions/base_requester" - path: v1/recents + path: api/v2/organizations http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" - limit: "50" - items: organization + include_fields: >- + ["next_activity_id","last_activity_id","open_deals_count","related_open_deals_count","closed_deals_count","related_closed_deals_count","email_messages_count","people_count","activities_count","done_activities_count","undone_activities_count","files_count","notes_count","followers_count","won_deals_count","related_won_deals_count","lost_deals_count","related_lost_deals_count"] record_selector: type: RecordSelector extractor: @@ -525,24 +511,40 @@ definitions: page_token_option: type: RequestOption inject_into: request_parameter - field_name: start + field_name: cursor + page_size_option: + type: RequestOption + inject_into: request_parameter + field_name: limit pagination_strategy: type: CursorPagination - cursor_value: "{{ response['additional_data']['pagination']['next_start'] }}" + page_size: 5 + cursor_value: "{{ response.get(\"additional_data\", {}).get(\"next_cursor\", {}) }}" + stop_condition: >- + {{ not response.get("additional_data", {}).get("next_cursor", {}) + }} incremental_sync: type: DatetimeBasedCursor cursor_field: update_time cursor_datetime_formats: - - "%Y-%m-%d %H:%M:%S" - datetime_format: "%Y-%m-%d %H:%M:%S" + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_datetime: type: MinMaxDatetime - datetime: "{{ config['replication_start_date'] }}" + datetime: "{{ config[\"replication_start_date\"] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption inject_into: request_parameter - field_name: since_timestamp + field_name: updated_since + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: updated_until + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" schema_loader: type: InlineSchemaLoader schema: @@ -557,7 +559,6 @@ definitions: path: v1/organizationFields http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -588,7 +589,6 @@ definitions: path: v1/permissionSets http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -612,7 +612,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: person record_selector: @@ -663,7 +662,6 @@ definitions: path: v1/personFields http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -696,7 +694,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: pipeline record_selector: @@ -749,7 +746,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: product record_selector: @@ -800,7 +796,6 @@ definitions: path: v1/productFields http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -831,7 +826,6 @@ definitions: path: v1/roles http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -864,7 +858,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: stage record_selector: @@ -917,7 +910,6 @@ definitions: path: v1/recents http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" items: user record_selector: @@ -968,7 +960,6 @@ definitions: path: v1/deals/{{ stream_slice.parent_id }}/products http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -1009,7 +1000,6 @@ definitions: path: v1/mailbox/mailThreads http_method: GET request_parameters: - api_token: "{{ config['api_token'] }}" limit: "50" record_selector: type: RecordSelector @@ -1045,6 +1035,13 @@ definitions: base_requester: type: HttpRequester url_base: https://api.pipedrive.com/ + authenticator: + type: ApiKeyAuthenticator + inject_into: + type: RequestOption + inject_into: request_parameter + field_name: api_token + api_token: "{{ config[\"api_token\"] }}" streams: - $ref: "#/definitions/streams/deals" From c75f1a190b1c99e46a7f3d0cd4eb4f6458d0ab6b Mon Sep 17 00:00:00 2001 From: Tope Folorunso Date: Tue, 18 Mar 2025 09:11:51 +0100 Subject: [PATCH 07/11] fix --- airbyte-integrations/connectors/source-pipedrive/manifest.yaml | 2 +- airbyte-integrations/connectors/source-pipedrive/metadata.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-pipedrive/manifest.yaml b/airbyte-integrations/connectors/source-pipedrive/manifest.yaml index c6fd2a2af5d20..f781f6a3d6a44 100644 --- a/airbyte-integrations/connectors/source-pipedrive/manifest.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/manifest.yaml @@ -1,4 +1,4 @@ -version: 6.39.3 +version: 6.41.1 type: DeclarativeSource diff --git a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml index 6ce046d1d9d48..b43a3c5cc148e 100644 --- a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml @@ -63,5 +63,5 @@ data: type: GSM alias: airbyte-connector-testing-secret-store connectorBuildOptions: - baseImage: docker.io/airbyte/source-declarative-manifest:6.33.1@sha256:06468f2b0acdb0126a29757f67025f8f837014f70e3f079e10e304b0e1a6be4b + baseImage: docker.io/airbyte/source-declarative-manifest:6.41.1@sha256:f256b6f008ec07a37e3e6bce3ada17bec4197c1bc66c41c4b756980c8cb587ac metadataSpecVersion: "1.0" From 2857543b6d7692f1c2607fabb74f37d233b6fcba Mon Sep 17 00:00:00 2001 From: Tope Folorunso Date: Tue, 18 Mar 2025 12:35:43 +0100 Subject: [PATCH 08/11] fix component --- .../connectors/source-pipedrive/components.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-pipedrive/components.py b/airbyte-integrations/connectors/source-pipedrive/components.py index 2f34fa080e105..cf3aa4231e08e 100644 --- a/airbyte-integrations/connectors/source-pipedrive/components.py +++ b/airbyte-integrations/connectors/source-pipedrive/components.py @@ -1,7 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from dataclasses import InitVar, dataclass +from dataclasses import InitVar, dataclass, field from typing import Any, List, Mapping, Union import requests @@ -41,7 +41,7 @@ class NullCheckedDpathExtractor(RecordExtractor): nullable_nested_field: Union[InterpolatedString, str] config: Config parameters: InitVar[Mapping[str, Any]] - decoder: Decoder = JsonDecoder(parameters={}) + decoder: Decoder = field(default_factory=lambda: JsonDecoder(parameters={})) def __post_init__(self, parameters: Mapping[str, Any]): self._dpath_extractor = DpathExtractor( From 7d0f93c4a7329c0b385e56c4f20c65b255fe1c7e Mon Sep 17 00:00:00 2001 From: Tope Folorunso Date: Tue, 18 Mar 2025 12:49:42 +0100 Subject: [PATCH 09/11] enable incremental test --- .../acceptance-test-config.yml | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml index 9b2d767f23bb4..37b58c532ef53 100644 --- a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml @@ -52,7 +52,33 @@ acceptance_tests: - name: persons incremental: - bypass_reason: "All incremental streams are empty in sandbox account." + tests: + - config_path: secrets/config.json + configured_catalog_path: integration_tests/configured_catalog.json + future_state: + future_state_path: integration_tests/abnormal_state.json + missing_streams: + - name: deals + bypass_reason: "Stream is empty in sandbox account." + - name: files + bypass_reason: "Stream is empty in sandbox account." + - name: filters + bypass_reason: "Stream is empty in sandbox account." + - name: notes + bypass_reason: "Stream is empty in sandbox account." + - name: activities + bypass_reason: "Stream is empty in sandbox account." + - name: persons + bypass_reason: "Stream is empty in sandbox account." + - name: pipelines + bypass_reason: "Stream is empty in sandbox account." + - name: products + bypass_reason: "Stream is empty in sandbox account." + - name: stages + bypass_reason: "Stream is empty in sandbox account." + - name: users + bypass_reason: "Stream is empty in sandbox account." + skip_comprehensive_incremental_tests: true full_refresh: tests: - config_path: "secrets/config.json" From e86b4c5bc48466d7c9cfe55aea1a579b044f650e Mon Sep 17 00:00:00 2001 From: Tope Folorunso Date: Tue, 18 Mar 2025 13:23:04 +0100 Subject: [PATCH 10/11] fix schema --- .../connectors/source-pipedrive/manifest.yaml | 235 ++++-------------- 1 file changed, 43 insertions(+), 192 deletions(-) diff --git a/airbyte-integrations/connectors/source-pipedrive/manifest.yaml b/airbyte-integrations/connectors/source-pipedrive/manifest.yaml index f781f6a3d6a44..3b407cfe381ce 100644 --- a/airbyte-integrations/connectors/source-pipedrive/manifest.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/manifest.yaml @@ -2816,231 +2816,90 @@ schemas: $schema: http://json-schema.org/draft-07/schema# additionalProperties: true properties: - active_flag: - type: - - "null" - - boolean - activities_count: - type: - - "null" - - number add_time: type: - "null" - string address: - type: - - "null" - - string - address_admin_area_level_1: - type: - - "null" - - string - address_admin_area_level_2: - type: - - "null" - - string - address_country: - type: - - "null" - - string - address_formatted_address: - type: - - "null" - - string - address_locality: - type: - - "null" - - string - address_postal_code: - type: - - "null" - - string - address_route: - type: - - "null" - - string - address_street_number: - type: - - "null" - - string - address_sublocality: - type: - - "null" - - string - address_subpremise: - type: - - "null" - - string - category_id: - type: - - "null" - - integer - cc_email: - type: - - "null" - - string - closed_deals_count: - type: - - "null" - - number - company_id: - type: - - "null" - - number - country_code: - type: - - "null" - - string - delete_time: - type: - - "null" - - string - done_activities_count: - type: - - "null" - - number - email_messages_count: - type: - - "null" - - number - files_count: - type: - - "null" - - number - first_char: - type: - - "null" - - string - followers_count: - type: - - "null" - - number - id: - type: - - "null" - - number - label: - type: - - "null" - - number - last_activity_date: - type: - - "null" - - string - last_activity_id: - type: - - "null" - - number - lost_deals_count: - type: - - "null" - - number - name: - type: - - "null" - - string - next_activity_date: - type: - - "null" - - string - next_activity_id: - type: - - "null" - - number - next_activity_time: - type: - - "null" - - string - notes_count: - type: - - "null" - - number - open_deals_count: - type: - - "null" - - number - owner_id: type: - "null" - object - - number properties: - active_flag: + value: type: - "null" - - boolean - email: + - string + country: type: - "null" - string - has_pic: + admin_area_level_1: type: - "null" - - number - id: + - string + admin_area_level_2: type: - "null" - - number - name: + - string + locality: type: - "null" - string - pic_hash: + sublocality: type: - "null" - string - value: + route: type: - "null" - - number - owner_name: + - string + street_number: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + formatted_address: + type: + - "null" + - string + id: type: - "null" - - string - people_count: + - number + name: type: - "null" - - number - picture_id: + - string + owner_id: type: - "null" - object + - number properties: active_flag: type: - "null" - boolean - add_time: + email: type: - "null" - string - added_by_user_id: + has_pic: type: - "null" - number - item_id: + id: type: - "null" - number - item_type: + name: type: - "null" - string - pictures: - type: - - "null" - - object - properties: - "128": - type: - - "null" - - string - "512": - type: - - "null" - - string - update_time: + pic_hash: type: - "null" - string @@ -3048,26 +2907,22 @@ schemas: type: - "null" - number - related_closed_deals_count: - type: - - "null" - - number - related_lost_deals_count: - type: - - "null" - - number - related_open_deals_count: + org_id: type: - "null" - number - related_won_deals_count: + is_deleted: type: - "null" - - number - undone_activities_count: + - boolean + label_ids: type: - "null" - - number + - array + items: + type: + - "null" + - number update_time: type: - "null" @@ -3077,10 +2932,6 @@ schemas: - "null" - number - string - won_deals_count: - type: - - "null" - - number organization_fields: type: object $schema: http://json-schema.org/draft-07/schema# From d44e2c13cd3a8d9b69d1cfcd9e3f791bcff6a84d Mon Sep 17 00:00:00 2001 From: Tope Folorunso Date: Tue, 18 Mar 2025 14:19:28 +0100 Subject: [PATCH 11/11] fix --- .../connectors/source-pipedrive/acceptance-test-config.yml | 3 +++ .../source-pipedrive/integration_tests/abnormal_state.json | 2 +- airbyte-integrations/connectors/source-pipedrive/manifest.yaml | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml index 37b58c532ef53..c65a89e4057dd 100644 --- a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml @@ -30,6 +30,9 @@ acceptance_tests: - name: stages - name: deal_products - name: mail + - name: deals + - name: users + - name: persons - config_path: "secrets/config.json" expect_records: path: "integration_tests/expected_records.jsonl" diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/abnormal_state.json index ec68ed9e2a854..dcee6c38d6c02 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/abnormal_state.json @@ -124,7 +124,7 @@ "type": "STREAM", "stream": { "stream_state": { - "update_time": "2217-06-26 21:20:07" + "update_time": "2217-06-26T21:20:07Z" }, "stream_descriptor": { "name": "organizations" diff --git a/airbyte-integrations/connectors/source-pipedrive/manifest.yaml b/airbyte-integrations/connectors/source-pipedrive/manifest.yaml index 3b407cfe381ce..bc6ebee86aead 100644 --- a/airbyte-integrations/connectors/source-pipedrive/manifest.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/manifest.yaml @@ -518,7 +518,7 @@ definitions: field_name: limit pagination_strategy: type: CursorPagination - page_size: 5 + page_size: 100 cursor_value: "{{ response.get(\"additional_data\", {}).get(\"next_cursor\", {}) }}" stop_condition: >- {{ not response.get("additional_data", {}).get("next_cursor", {})