tmp: tuning knobs

Author: johnny-schmidt

airbyte-integrations/connectors/destination-mssql-v2/src/main/kotlin/io/airbyte/integrations/destination/mssql/v2/MSSQLBulkLoader.kt

@@ -109,8 +109,8 @@ class MSSQLBulkLoaderFactory(
     private val bulkLoadConfig: MSSQLBulkLoadConfiguration,
     private val streamStateStore: StreamStateStore<MSSQLStreamState>
 ) : BulkLoaderFactory<StreamKey, AzureBlob> {
-    override val numPartWorkers: Int = 2
-    override val numUploadWorkers: Int = 10
+    override val numPartWorkers: Int = config.numProcessRecordsWorkers
+    override val numUploadWorkers: Int = config.numProcessBatchWorkers
     override val maxNumConcurrentLoads: Int = 1
 
     override val objectSizeBytes: Long = 200 * 1024 * 1024

airbyte-integrations/connectors/destination-mssql-v2/src/main/kotlin/io/airbyte/integrations/destination/mssql/v2/config/MSSQLConfiguration.kt

@@ -22,9 +22,9 @@ data class MSSQLConfiguration(
     val jdbcUrlParams: String?,
     val sslMethod: EncryptionMethod,
     override val mssqlLoadTypeConfiguration: MSSQLLoadTypeConfiguration,
-) : DestinationConfiguration(), MSSQLLoadTypeConfigurationProvider {
-    override val numProcessRecordsWorkers = 1
+    override val numProcessRecordsWorkers: Int = 1,
     override val numProcessBatchWorkers: Int = 1
+) : DestinationConfiguration(), MSSQLLoadTypeConfigurationProvider {
     override val processEmptyFiles: Boolean = true
     override val recordBatchSizeBytes = ObjectStorageUploadConfiguration.DEFAULT_PART_SIZE_BYTES
 }
@@ -58,7 +58,9 @@ class MSSQLConfigurationFactory(private val featureFlags: Set<FeatureFlag>) :
             password = overrides.getOrDefault("password", spec.password),
             jdbcUrlParams = overrides.getOrDefault("jdbcUrlParams", spec.jdbcUrlParams),
             sslMethod = spec.sslMethod,
-            mssqlLoadTypeConfiguration = spec.toLoadConfiguration()
+            mssqlLoadTypeConfiguration = spec.toLoadConfiguration(),
+            numProcessRecordsWorkers = spec.numPartWorkers ?: 2,
+            numProcessBatchWorkers = spec.numObjectLoaders ?: 10
         )
     }
 }

airbyte-integrations/connectors/destination-mssql-v2/src/main/kotlin/io/airbyte/integrations/destination/mssql/v2/config/MSSQLSpecification.kt

@@ -78,6 +78,12 @@ class MSSQLSpecification : ConfigurationSpecification(), LoadTypeSpecification {
     @get:JsonProperty("load_type")
     @get:JsonSchemaInject(json = """{"always_show": true,"order":8}""")
     override val loadType: LoadType = InsertLoadSpecification()
+
+    @get:JsonProperty("numPartWorkers")
+    val numPartWorkers: Int? = null
+
+    @get:JsonProperty("numObjectLoaders")
+    val numObjectLoaders: Int? = null
 }
 
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "name")