Skip to content

Commit aa5e618

Browse files
committed
Compilation errors
1 parent 82180ae commit aa5e618

File tree

1 file changed

+36
-24
lines changed

1 file changed

+36
-24
lines changed

airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/InputConsumerTask.kt

+36-24
Original file line numberDiff line numberDiff line change
@@ -95,9 +95,9 @@ class DefaultInputConsumerTask(
9595
reserved: Reserved<DestinationStreamAffinedMessage>,
9696
sizeBytes: Long
9797
) {
98-
val stream = reserved.value.stream
99-
val manager = syncManager.getStreamManager(stream)
100-
val recordQueue = recordQueueSupplier.get(stream)
98+
val streamDescriptor = reserved.value.stream.descriptor
99+
val manager = syncManager.getStreamManager(streamDescriptor)
100+
val recordQueue = recordQueueSupplier.get(streamDescriptor)
101101
when (val message = reserved.value) {
102102
is DestinationRecord -> {
103103
val wrapped =
@@ -111,93 +111,105 @@ class DefaultInputConsumerTask(
111111
is DestinationRecordStreamComplete -> {
112112
reserved.release() // safe because multiple calls conflate
113113
val wrapped = StreamEndEvent(index = manager.markEndOfStream(true))
114-
log.info { "Read COMPLETE for stream $stream" }
114+
log.info { "Read COMPLETE for stream $streamDescriptor" }
115115
recordQueue.publish(reserved.replace(wrapped))
116116
recordQueue.close()
117117
}
118118
is DestinationRecordStreamIncomplete -> {
119119
reserved.release() // safe because multiple calls conflate
120120
val wrapped = StreamEndEvent(index = manager.markEndOfStream(false))
121-
log.info { "Read INCOMPLETE for stream $stream" }
121+
log.info { "Read INCOMPLETE for stream $streamDescriptor" }
122122
recordQueue.publish(reserved.replace(wrapped))
123123
recordQueue.close()
124124
}
125125
is DestinationFile -> {
126126
val index = manager.incrementReadCount()
127127
// destinationTaskLauncher.handleFile(stream, message, index)
128-
fileTransferQueue.publish(FileTransferQueueMessage(stream, message, index))
128+
fileTransferQueue.publish(
129+
FileTransferQueueMessage(streamDescriptor, message, index)
130+
)
129131
}
130132
is DestinationFileStreamComplete -> {
131133
reserved.release() // safe because multiple calls conflate
132134
manager.markEndOfStream(true)
133135
val envelope =
134136
BatchEnvelope(
135137
SimpleBatch(Batch.State.COMPLETE),
136-
streamDescriptor = message.stream,
138+
streamDescriptor = streamDescriptor,
137139
)
138-
destinationTaskLauncher.handleNewBatch(stream, envelope)
140+
destinationTaskLauncher.handleNewBatch(streamDescriptor, envelope)
139141
}
140142
is DestinationFileStreamIncomplete ->
141-
throw IllegalStateException("File stream $stream failed upstream, cannot continue.")
143+
throw IllegalStateException(
144+
"File stream $streamDescriptor failed upstream, cannot continue."
145+
)
142146
}
143147
}
144148

145149
private suspend fun handleRecordForPipeline(
146150
reserved: Reserved<DestinationStreamAffinedMessage>,
147151
) {
148-
val stream = reserved.value.stream
149-
unopenedStreams.remove(stream)?.let {
150-
log.info { "Saw first record for stream $stream; initializing" }
152+
val streamDescriptor = reserved.value.stream.descriptor
153+
unopenedStreams.remove(streamDescriptor)?.let {
154+
log.info { "Saw first record for stream $streamDescriptor; initializing" }
151155
// Note, since we're not spilling to disk, there is nothing to do with
152156
// any records before initialization is complete, so we'll wait here
153157
// for it to finish.
154158
openStreamQueue.publish(it)
155-
syncManager.getOrAwaitStreamLoader(stream)
156-
log.info { "Initialization for stream $stream complete" }
159+
syncManager.getOrAwaitStreamLoader(streamDescriptor)
160+
log.info { "Initialization for stream $streamDescriptor complete" }
157161
}
158-
val manager = syncManager.getStreamManager(stream)
162+
val manager = syncManager.getStreamManager(streamDescriptor)
159163
when (val message = reserved.value) {
160164
is DestinationRecord -> {
161165
val record = message.asDestinationRecordRaw()
162166
manager.incrementReadCount()
163167
val pipelineMessage =
164168
PipelineMessage(
165169
mapOf(manager.getCurrentCheckpointId() to 1),
166-
StreamKey(stream),
170+
StreamKey(streamDescriptor),
167171
record
168172
)
169173
val partition = partitioner.getPartition(record, recordQueueForPipeline.partitions)
170174
recordQueueForPipeline.publish(reserved.replace(pipelineMessage), partition)
171175
}
172176
is DestinationRecordStreamComplete -> {
173177
manager.markEndOfStream(true)
174-
log.info { "Read COMPLETE for stream $stream" }
175-
recordQueueForPipeline.broadcast(reserved.replace(PipelineEndOfStream(stream)))
178+
log.info { "Read COMPLETE for stream $streamDescriptor" }
179+
recordQueueForPipeline.broadcast(
180+
reserved.replace(PipelineEndOfStream(streamDescriptor))
181+
)
176182
reserved.release()
177183
}
178184
is DestinationRecordStreamIncomplete -> {
179185
manager.markEndOfStream(false)
180-
log.info { "Read INCOMPLETE for stream $stream" }
181-
recordQueueForPipeline.broadcast(reserved.replace(PipelineEndOfStream(stream)))
186+
log.info { "Read INCOMPLETE for stream $streamDescriptor" }
187+
recordQueueForPipeline.broadcast(
188+
reserved.replace(PipelineEndOfStream(streamDescriptor))
189+
)
182190
reserved.release()
183191
}
184192
is DestinationFile -> {
185193
val index = manager.incrementReadCount()
186194
// destinationTaskLauncher.handleFile(stream, message, index)
187-
fileTransferQueue.publish(FileTransferQueueMessage(stream, message, index))
195+
fileTransferQueue.publish(
196+
FileTransferQueueMessage(streamDescriptor, message, index)
197+
)
188198
}
189199
is DestinationFileStreamComplete -> {
190200
reserved.release() // safe because multiple calls conflate
191201
manager.markEndOfStream(true)
192202
val envelope =
193203
BatchEnvelope(
194204
SimpleBatch(Batch.State.COMPLETE),
195-
streamDescriptor = message.stream,
205+
streamDescriptor = streamDescriptor,
196206
)
197-
destinationTaskLauncher.handleNewBatch(stream, envelope)
207+
destinationTaskLauncher.handleNewBatch(streamDescriptor, envelope)
198208
}
199209
is DestinationFileStreamIncomplete ->
200-
throw IllegalStateException("File stream $stream failed upstream, cannot continue.")
210+
throw IllegalStateException(
211+
"File stream $streamDescriptor failed upstream, cannot continue."
212+
)
201213
}
202214
}
203215

0 commit comments

Comments
 (0)