Skip to content

Commit 4efc065

Browse files
edgaogisripa
edgao
and
gisripa
authored
DV2 destinations: Build DestinationState / Migration framework (#35303)
Signed-off-by: Gireesh Sreepathi <[email protected]> Co-authored-by: Gireesh Sreepathi <[email protected]>
1 parent 7277fc5 commit 4efc065

File tree

24 files changed

+908
-222
lines changed

24 files changed

+908
-222
lines changed

‎airbyte-cdk/java/airbyte-cdk/README.md

+3-2
Original file line numberDiff line numberDiff line change
@@ -166,9 +166,10 @@ MavenLocal debugging steps:
166166

167167
| Version | Date | Pull Request | Subject |
168168
|:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
169-
| 0.23.9 | 2024-03-01 | [\#35720](https://github.com/airbytehq/airbyte/pull/35720) | various improvements for tests TestDataHolder |
169+
| 0.23.10 | 2024-03-01 | [\#35303](https://github.com/airbytehq/airbyte/pull/35303) | various improvements for tests TestDataHolder |
170+
| 0.23.9 | 2024-03-01 | [\#35720](https://github.com/airbytehq/airbyte/pull/35720) | various improvements for tests TestDataHolder |
170171
| 0.23.8 | 2024-02-28 | [\#35529](https://github.com/airbytehq/airbyte/pull/35529) | Refactor on state iterators |
171-
| 0.23.7 | 2024-02-28 | [\#35376](https://github.com/airbytehq/airbyte/pull/35376) | Extract typereduper migrations to separte method |
172+
| 0.23.7 | 2024-02-28 | [\#35376](https://github.com/airbytehq/airbyte/pull/35376) | Extract typereduper migrations to separte method |
172173
| 0.23.6 | 2024-02-26 | [\#35647](https://github.com/airbytehq/airbyte/pull/35647) | Add a getNamespace into TestDataHolder |
173174
| 0.23.5 | 2024-02-26 | [\#35512](https://github.com/airbytehq/airbyte/pull/35512) | Remove @DisplayName from all CDK tests. |
174175
| 0.23.4 | 2024-02-26 | [\#35507](https://github.com/airbytehq/airbyte/pull/35507) | Add more logs into TestDatabase. |

‎airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version=0.23.9
1+
version=0.23.10

‎airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java

+13-7
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
package io.airbyte.cdk.integrations.destination.jdbc;
66

7+
import static io.airbyte.cdk.integrations.base.JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE;
78
import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage;
89
import static io.airbyte.cdk.integrations.util.ConfiguredCatalogUtilKt.addDefaultNamespaceToStreams;
910

@@ -17,7 +18,6 @@
1718
import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer;
1819
import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility;
1920
import io.airbyte.cdk.integrations.base.Destination;
20-
import io.airbyte.cdk.integrations.base.JavaBaseConstants;
2121
import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer;
2222
import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag;
2323
import io.airbyte.cdk.integrations.destination.NamingConventionTransformer;
@@ -37,6 +37,7 @@
3737
import io.airbyte.integrations.base.destination.typing_deduping.NoopV2TableMigrator;
3838
import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog;
3939
import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper;
40+
import io.airbyte.integrations.base.destination.typing_deduping.migrators.MinimumDestinationState;
4041
import io.airbyte.protocol.models.v0.AirbyteConnectionStatus;
4142
import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status;
4243
import io.airbyte.protocol.models.v0.AirbyteMessage;
@@ -45,6 +46,7 @@
4546
import java.util.List;
4647
import java.util.Map;
4748
import java.util.Objects;
49+
import java.util.Optional;
4850
import java.util.UUID;
4951
import java.util.function.Consumer;
5052
import javax.sql.DataSource;
@@ -93,7 +95,7 @@ public AirbyteConnectionStatus check(final JsonNode config) {
9395
attemptTableOperations(outputSchema, database, namingResolver, sqlOperations, false);
9496
if (TypingAndDedupingFlag.isDestinationV2()) {
9597
final var v2RawSchema = namingResolver.getIdentifier(TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE)
96-
.orElse(JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE));
98+
.orElse(DEFAULT_AIRBYTE_INTERNAL_NAMESPACE));
9799
attemptTableOperations(v2RawSchema, database, namingResolver, sqlOperations, false);
98100
destinationSpecificTableOperations(database);
99101
}
@@ -252,7 +254,9 @@ private void assertCustomParametersDontOverwriteDefaultParameters(final Map<Stri
252254

253255
protected abstract JdbcSqlGenerator getSqlGenerator();
254256

255-
protected abstract JdbcDestinationHandler getDestinationHandler(final String databaseName, final JdbcDatabase database);
257+
protected abstract JdbcDestinationHandler<? extends MinimumDestinationState> getDestinationHandler(final String databaseName,
258+
final JdbcDatabase database,
259+
final String rawTableSchema);
256260

257261
/**
258262
* "database" key at root of the config json, for any other variants in config, override this
@@ -309,21 +313,23 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN
309313
*/
310314
private TyperDeduper getV2TyperDeduper(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final JdbcDatabase database) {
311315
final JdbcSqlGenerator sqlGenerator = getSqlGenerator();
312-
final ParsedCatalog parsedCatalog = TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE)
316+
Optional<String> rawNamespaceOverride = TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE);
317+
final ParsedCatalog parsedCatalog = rawNamespaceOverride
313318
.map(override -> new CatalogParser(sqlGenerator, override))
314319
.orElse(new CatalogParser(sqlGenerator))
315320
.parseCatalog(catalog);
316321
final String databaseName = getDatabaseName(config);
317322
final var migrator = new JdbcV1V2Migrator(namingResolver, database, databaseName);
318323
final NoopV2TableMigrator v2TableMigrator = new NoopV2TableMigrator();
319-
final DestinationHandler destinationHandler = getDestinationHandler(databaseName, database);
324+
final DestinationHandler<? extends MinimumDestinationState> destinationHandler =
325+
getDestinationHandler(databaseName, database, rawNamespaceOverride.orElse(DEFAULT_AIRBYTE_INTERNAL_NAMESPACE));
320326
final boolean disableTypeDedupe = config.has(DISABLE_TYPE_DEDUPE) && config.get(DISABLE_TYPE_DEDUPE).asBoolean(false);
321327
final TyperDeduper typerDeduper;
322328
if (disableTypeDedupe) {
323-
typerDeduper = new NoOpTyperDeduperWithV1V2Migrations(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator);
329+
typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, List.of());
324330
} else {
325331
typerDeduper =
326-
new DefaultTyperDeduper(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator);
332+
new DefaultTyperDeduper<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, List.of());
327333
}
328334
return typerDeduper;
329335
}

0 commit comments

Comments
 (0)