feat(source-airtable): migrate to low-code (#49813)

Co-authored-by: alafanechere <[email protected]>
Co-authored-by: Serhii Lazebnyi <[email protected]>
Co-authored-by: Octavia Squidington III <[email protected]>

Author: 4 people

airbyte-integrations/connectors/source-airtable/.coveragerc

@@ -0,0 +1,3 @@
+[run]
+omit =
+    source_airtable/run.py

airbyte-integrations/connectors/source-airtable/acceptance-test-config.yml

@@ -5,7 +5,7 @@ test_strictness_level: high
 acceptance_tests:
   spec:
     tests:
-      - spec_path: "source_airtable/spec.json"
+      - spec_path: "integration_tests/spec.json"
   connection:
     tests:
       - config_path: "secrets/config.json"

airbyte-integrations/connectors/source-airtable/source_airtable/spec.json airbyte-integrations/connectors/source-airtable/integration_tests/spec.json

@@ -1,10 +1,7 @@
 {
-  "documentationUrl": "https://docs.airbyte.com/integrations/sources/airtable",
   "connectionSpecification": {
-    "$schema": "https://json-schema.org/draft-07/schema#",
-    "title": "Airtable Source Spec",
     "type": "object",
-    "additionalProperties": true,
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "properties": {
       "credentials": {
         "title": "Authentication",
@@ -27,8 +24,8 @@
               },
               "client_secret": {
                 "type": "string",
-                "title": "Client secret",
-                "description": "The client secret the Airtable developer application.",
+                "title": "Client Secret",
+                "description": "The client secret of the Airtable developer application.",
                 "airbyte_secret": true
               },
               "access_token": {
@@ -50,8 +47,8 @@
             }
           },
           {
-            "title": "Personal Access Token",
             "type": "object",
+            "title": "Personal Access Token",
             "required": ["api_key"],
             "properties": {
               "auth_method": {
@@ -69,8 +66,11 @@
           }
         ]
       }
-    }
+    },
+    "additionalProperties": true
   },
+  "supportsNormalization": false,
+  "supportsDBT": false,
   "advanced_auth": {
     "auth_flow_type": "oauth2.0",
     "predicate_key": ["credentials", "auth_method"],

airbyte-integrations/connectors/source-airtable/metadata.yaml

@@ -7,11 +7,11 @@ data:
       - api.airtable.com
       - airtable.com
   connectorBuildOptions:
-    baseImage: docker.io/airbyte/python-connector-base:2.0.0@sha256:c44839ba84406116e8ba68722a0f30e8f6e7056c726f447681bb9e9ece8bd916
+    baseImage: docker.io/airbyte/python-connector-base:3.0.0@sha256:1a0845ff2b30eafa793c6eee4e8f4283c2e52e1bbd44eed6cb9e9abd5d34d844
   connectorSubtype: api
   connectorType: source
   definitionId: 14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212
-  dockerImageTag: 4.4.0
+  dockerImageTag: 4.5.0-rc.1
   dockerRepository: airbyte/source-airtable
   documentationUrl: https://docs.airbyte.com/integrations/sources/airtable
   githubIssueLabel: source-airtable
@@ -34,10 +34,12 @@ data:
       4.0.0:
         message: This release introduces changes to columns with formula to parse values directly from `array` to `string` or `number` (where it is possible). Users should refresh the source schema and reset affected streams after upgrading to ensure uninterrupted syncs.
         upgradeDeadline: "2023-10-23"
+    rolloutConfiguration:
+      enableProgressiveRollout: true
   supportLevel: certified
   tags:
     - language:python
-    - cdk:python
+    - cdk:low-code
   connectorTestSuitesOptions:
     - suite: liveTests
       testConnections:

airbyte-integrations/connectors/source-airtable/poetry.lock

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "4.4.0"
+version = "4.5.0-rc.1"
 name = "source-airtable"
 description = "Source implementation for Airtable."
 authors = [ "Airbyte <[email protected]>",]
@@ -16,8 +16,8 @@ repository = "https://github.com/airbytehq/airbyte"
 include = "source_airtable"
 
 [tool.poetry.dependencies]
-python = "^3.9,<3.12"
-airbyte-cdk = "^3"
+python = "^3.10,<3.12"
+airbyte-cdk = "^6"
 
 [tool.poetry.scripts]
 source-airtable = "source_airtable.run:run"

airbyte-integrations/connectors/source-airtable/pyproject.toml

@@ -0,0 +1,79 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from typing import Any, List, Mapping
+
+import orjson
+
+from airbyte_cdk.config_observation import create_connector_config_control_message
+from airbyte_cdk.entrypoint import AirbyteEntrypoint
+from airbyte_cdk.models import AirbyteMessageSerializer
+from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
+from source_airtable import SourceAirtable
+
+
+class MigrateApiKey:
+    """
+    Migrates old configs to a new supported format.
+
+    OLD:
+    {
+    "api_key": ...
+    }
+
+    NEW:
+    {
+    "credentials": {"api_key": ..., "auth_method": "api_key"}
+    }
+
+    """
+
+    migrate_from_key = "api_key"
+    migrate_to_key = "credentials"
+
+    message_repository: MessageRepository = InMemoryMessageRepository()
+
+    @classmethod
+    def _should_migrate(cls, config: Mapping[str, Any]) -> bool:
+        if cls.migrate_from_key in config and cls.migrate_to_key not in config:
+            return True
+        return False
+
+    @classmethod
+    def _migrate_api_key(cls, config: Mapping[str, Any]) -> Mapping[str, Any]:
+        config[cls.migrate_to_key] = {cls.migrate_from_key: config[cls.migrate_from_key], "auth_method": "api_key"}
+        return config
+
+    @classmethod
+    def _modify_and_save(cls, config_path: str, source: SourceAirtable, config: Mapping[str, Any]) -> Mapping[str, Any]:
+        # modify the config
+        migrated_config = cls._migrate_api_key(config)
+        # save the config
+        source.write_config(migrated_config, config_path)
+        # return modified config
+        return migrated_config
+
+    @classmethod
+    def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None:
+        # add the Airbyte Control Message to message repo
+        cls.message_repository.emit_message(create_connector_config_control_message(migrated_config))
+        # emit the Airbyte Control Message from message queue to stdout
+        for message in cls.message_repository._message_queue:
+            print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())
+
+    @classmethod
+    def migrate(cls, args: List[str], source: SourceAirtable) -> None:
+        """
+        This method checks the input args, should the config be migrated,
+        transform if necessary and emit the CONTROL message.
+        """
+        # get config path
+        config_path = AirbyteEntrypoint(source).extract_config(args)
+        # proceed only if `--config` arg is provided
+        if config_path:
+            # read the existing config
+            config = source.read_config(config_path)
+            # migration check
+            if cls._should_migrate(config):
+                cls._emit_control_message(
+                    cls._modify_and_save(config_path, source, config),
+                )