Skip to content

Commit c494847

Browse files
feat(source-s3): bump cdk to have preserve sub directories option in copy raw files (file-transfer) (#51474)
1 parent d4c848c commit c494847

File tree

6 files changed

+166
-260
lines changed

6 files changed

+166
-260
lines changed

airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json

+6
Original file line numberDiff line numberDiff line change
@@ -405,6 +405,12 @@
405405
"const": "use_file_transfer",
406406
"enum": ["use_file_transfer"],
407407
"type": "string"
408+
},
409+
"preserve_directory_structure": {
410+
"default": true,
411+
"description": "If enabled, sends subdirectory folder structure along with source file names to the destination. Otherwise, files will be synced by their names only. This option is ignored when file-based replication is not enabled.",
412+
"title": "Preserve Sub-Directories in File Paths",
413+
"type": "boolean"
408414
}
409415
},
410416
"description": "Copy raw files without parsing their contents. Bits are copied into the destination exactly as they appeared in the source. Recommended for use with unstructured text data, non-text and compressed files.",

airbyte-integrations/connectors/source-s3/integration_tests/spec.json

+6
Original file line numberDiff line numberDiff line change
@@ -405,6 +405,12 @@
405405
"const": "use_file_transfer",
406406
"enum": ["use_file_transfer"],
407407
"type": "string"
408+
},
409+
"preserve_directory_structure": {
410+
"default": true,
411+
"description": "If enabled, sends subdirectory folder structure along with source file names to the destination. Otherwise, files will be synced by their names only. This option is ignored when file-based replication is not enabled.",
412+
"title": "Preserve Sub-Directories in File Paths",
413+
"type": "boolean"
408414
}
409415
},
410416
"description": "Copy raw files without parsing their contents. Bits are copied into the destination exactly as they appeared in the source. Recommended for use with unstructured text data, non-text and compressed files.",

airbyte-integrations/connectors/source-s3/metadata.yaml

+4-2
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ data:
1010
connectorSubtype: file
1111
connectorType: source
1212
definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2
13-
dockerImageTag: 4.11.4
13+
dockerImageTag: 4.12.0-rc.1
1414
dockerRepository: airbyte/source-s3
1515
documentationUrl: https://docs.airbyte.com/integrations/sources/s3
1616
githubIssueLabel: source-s3
@@ -30,7 +30,7 @@ data:
3030
releaseStage: generally_available
3131
releases:
3232
rolloutConfiguration:
33-
enableProgressiveRollout: false
33+
enableProgressiveRollout: true
3434
breakingChanges:
3535
4.0.0:
3636
message: UX improvement, multi-stream support and deprecation of some parsing features
@@ -47,6 +47,8 @@ data:
4747
- suite: unitTests
4848
- suite: liveTests
4949
testConnections:
50+
- name: s3_100_mb_dev_null
51+
id: 3dbae341-863a-43c0-bf00-d6a7473c7364
5052
- name: s3_v4_jsonl_config_dev_null
5153
id: 01f508b1-5690-47af-940c-a777d471762f
5254
- name: s3_v4_csv_skip_rows_no_header_config_dev_null

0 commit comments

Comments
 (0)