Skip to content

Commit 96865c4

Browse files
stephane-airbytestephane-airbyte
committed
fix failing tests
1 parent cc8ac8f commit 96865c4

File tree

17 files changed

+412
-193
lines changed

17 files changed

+412
-193
lines changed

‎airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DataSourceFactory.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -238,9 +238,7 @@ public DataSourceBuilder withPassword(final String password) {
238238
}
239239

240240
public DataSourceBuilder withPort(final Integer port) {
241-
if (port != null) {
242-
this.port = port;
243-
}
241+
this.port = port;
244242
return this;
245243
}
246244

‎airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DatabaseDriver.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ public enum DatabaseDriver {
1414
DB2("com.ibm.db2.jcc.DB2Driver", "jdbc:db2://%s:%d/%s"),
1515
STARBURST("io.trino.jdbc.TrinoDriver", "jdbc:trino://%s:%s/%s?SSL=true&source=airbyte"),
1616
MARIADB("org.mariadb.jdbc.Driver", "jdbc:mariadb://%s:%d/%s"),
17-
MSSQLSERVER("com.microsoft.sqlserver.jdbc.SQLServerDriver", "jdbc:sqlserver://%s:%d/%s"),
17+
MSSQLSERVER("com.microsoft.sqlserver.jdbc.SQLServerDriver", "jdbc:sqlserver://%s:%d;databaseName=%s"),
1818
MYSQL("com.mysql.cj.jdbc.Driver", "jdbc:mysql://%s:%d/%s"),
1919
ORACLE("oracle.jdbc.OracleDriver", "jdbc:oracle:thin:@%s:%d/%s"),
2020
VERTICA("com.vertica.jdbc.Driver", "jdbc:vertica://%s:%d/%s"),

‎airbyte-cdk/java/airbyte-cdk/core/src/main/resources/bastion/Dockerfile

+5-4
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,23 @@
1-
FROM ubuntu:18.04
1+
FROM ubuntu:24.04
22

33
RUN apt-get update && apt-get install -y openssh-server
44
RUN apt-get install -y apt-utils
5-
RUN mkdir /var/run/sshd
5+
RUN mkdir -p /var/run/sshd
66
RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
77
RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
88

99
RUN useradd -m -s /bin/bash sshuser
1010
RUN echo "sshuser:secret" | chpasswd
11+
RUN echo "root:secret" | chpasswd
1112

12-
RUN mkdir /var/bastion
13+
RUN mkdir -p /var/bastion
1314
RUN ssh-keygen -m PEM -t rsa -b 4096 -C "test-container-bastion" -P "" -f /var/bastion/id_rsa -q
1415
RUN install -D /var/bastion/id_rsa.pub /home/sshuser/.ssh/authorized_keys
1516

1617
RUN chown -R sshuser:sshuser /home/sshuser/.ssh
1718
RUN chmod 600 /home/sshuser/.ssh/authorized_keys
1819

19-
RUN mkdir /root/.ssh
20+
RUN mkdir -p /root/.ssh
2021

2122
RUN apt-get clean && \
2223
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

‎airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/extensions/LoggingInvocationInterceptor.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,9 @@ public Object invoke(Object proxy, Method method, Object[] args) throws Throwabl
5959
logLineSuffix = "instance creation for %s".formatted(invocationContext.getTargetClass());
6060
} else if (methodMatcher.matches()) {
6161
String interceptedEvent = methodMatcher.group(1);
62-
logLineSuffix = "execution of @%s method %s.%s".formatted(invocationContext.getExecutable().getDeclaringClass().getSimpleName(),
63-
interceptedEvent, invocationContext.getExecutable().getName());
62+
logLineSuffix = "execution of @%s method %s.%s".formatted(interceptedEvent,
63+
invocationContext.getExecutable().getDeclaringClass().getSimpleName(),
64+
invocationContext.getExecutable().getName());
6465
} else {
6566
logLineSuffix = "execution of unknown intercepted call %s".formatted(methodName);
6667
}

‎airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java

+26-5
Original file line numberDiff line numberDiff line change
@@ -10,29 +10,50 @@
1010
import com.fasterxml.jackson.databind.JsonNode;
1111
import com.google.common.collect.ImmutableMap;
1212
import io.airbyte.cdk.integrations.util.HostPortResolver;
13+
import io.airbyte.cdk.testutils.ContainerFactory;
1314
import io.airbyte.commons.json.Jsons;
1415
import java.io.IOException;
1516
import java.util.List;
1617
import java.util.Objects;
18+
import java.util.function.Consumer;
1719
import org.apache.commons.lang3.tuple.ImmutablePair;
1820
import org.testcontainers.containers.Container;
1921
import org.testcontainers.containers.GenericContainer;
2022
import org.testcontainers.containers.JdbcDatabaseContainer;
2123
import org.testcontainers.containers.Network;
2224
import org.testcontainers.images.builder.ImageFromDockerfile;
25+
import org.testcontainers.utility.DockerImageName;
2326

2427
public class SshBastionContainer implements AutoCloseable {
2528

29+
public static class SshBastionContainerFactory extends ContainerFactory<GenericContainer<?>> {
30+
31+
@Override
32+
protected GenericContainer<?> createNewContainer(DockerImageName imageName) {
33+
var container = new GenericContainer(new ImageFromDockerfile("bastion-test")
34+
.withFileFromClasspath("Dockerfile", "bastion/Dockerfile"))
35+
.withExposedPorts(22);
36+
return container;
37+
}
38+
39+
public GenericContainer exclusive(final Network network) {
40+
Consumer<GenericContainer<?>> imageModifier = c -> {
41+
c.withNetwork(network);
42+
};
43+
var container = super.exclusive("bastion-test", new NamedContainerModifierImpl<>("withNetwork", imageModifier));
44+
return container;
45+
}
46+
47+
}
48+
49+
private static final SshBastionContainerFactory factory = new SshBastionContainerFactory();
50+
2651
private static final String SSH_USER = "sshuser";
2752
private static final String SSH_PASSWORD = "secret";
2853
private GenericContainer bastion;
2954

3055
public void initAndStartBastion(final Network network) {
31-
bastion = new GenericContainer(
32-
new ImageFromDockerfile("bastion-test")
33-
.withFileFromClasspath("Dockerfile", "bastion/Dockerfile"))
34-
.withNetwork(network)
35-
.withExposedPorts(22);
56+
bastion = factory.exclusive(network);
3657
bastion.start();
3758
}
3859

‎airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/ContainerFactory.java

+82-33
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
package io.airbyte.cdk.testutils;
66

7+
import com.google.common.collect.Lists;
78
import io.airbyte.commons.logging.LoggingHelper;
89
import io.airbyte.commons.logging.MdcScope;
910
import java.lang.reflect.InvocationTargetException;
@@ -13,13 +14,13 @@
1314
import java.util.concurrent.ConcurrentHashMap;
1415
import java.util.concurrent.ConcurrentMap;
1516
import java.util.concurrent.atomic.AtomicInteger;
17+
import java.util.function.Consumer;
1618
import java.util.function.Supplier;
1719
import java.util.stream.Stream;
1820
import org.apache.commons.lang3.StringUtils;
1921
import org.slf4j.Logger;
2022
import org.slf4j.LoggerFactory;
2123
import org.testcontainers.containers.GenericContainer;
22-
import org.testcontainers.containers.JdbcDatabaseContainer;
2324
import org.testcontainers.containers.output.OutputFrame;
2425
import org.testcontainers.containers.output.Slf4jLogConsumer;
2526
import org.testcontainers.utility.DockerImageName;
@@ -28,11 +29,13 @@
2829
* ContainerFactory is the companion to {@link TestDatabase} and provides it with suitable
2930
* testcontainer instances.
3031
*/
31-
public abstract class ContainerFactory<C extends JdbcDatabaseContainer<?>> {
32+
public abstract class ContainerFactory<C extends GenericContainer<?>> {
3233

3334
static private final Logger LOGGER = LoggerFactory.getLogger(ContainerFactory.class);
3435

35-
private record ContainerKey(Class<? extends ContainerFactory> clazz, DockerImageName imageName, List<String> methods) {};
36+
private record ContainerKey<C extends GenericContainer<?>> (Class<? extends ContainerFactory> clazz,
37+
DockerImageName imageName,
38+
List<NamedContainerModifier<C>> methods) {};
3639

3740
private static class ContainerOrException {
3841

@@ -67,12 +70,12 @@ GenericContainer<?> container() {
6770

6871
}
6972

70-
private static final ConcurrentMap<ContainerKey, ContainerOrException> SHARED_CONTAINERS = new ConcurrentHashMap<>();
73+
private final ConcurrentMap<ContainerKey<C>, ContainerOrException> SHARED_CONTAINERS = new ConcurrentHashMap<>();
7174
private static final AtomicInteger containerId = new AtomicInteger(0);
7275

73-
private static final MdcScope.Builder getTestContainerLogMdcBuilder(DockerImageName imageName, List<String> methods) {
76+
private final MdcScope.Builder getTestContainerLogMdcBuilder(DockerImageName imageName, List<NamedContainerModifier<C>> containerModifiers) {
7477
return new MdcScope.Builder()
75-
.setLogPrefix("testcontainer %s (%s[%s]):".formatted(containerId.incrementAndGet(), imageName, StringUtils.join(methods, ",")))
78+
.setLogPrefix("testcontainer %s (%s[%s]):".formatted(containerId.incrementAndGet(), imageName, StringUtils.join(containerModifiers, ",")))
7679
.setPrefixColor(LoggingHelper.Color.RED_BACKGROUND);
7780
}
7881

@@ -85,9 +88,21 @@ private static final MdcScope.Builder getTestContainerLogMdcBuilder(DockerImageN
8588
/**
8689
* Returns a shared instance of the testcontainer.
8790
*/
88-
@SuppressWarnings("unchecked")
91+
@Deprecated
8992
public final C shared(String imageName, String... methods) {
90-
final var containerKey = new ContainerKey(getClass(), DockerImageName.parse(imageName), Stream.of(methods).toList());
93+
return shared(imageName, (NamedContainerModifier<C>) Stream.of(methods).map(n -> new NamedContainerModifierImpl<C>(n, resolveModifierByName(n))).toList());
94+
}
95+
96+
public final C shared(String imageName, NamedContainerModifier<C>... namedContainerModifiers) {
97+
return shared(imageName, List.of(namedContainerModifiers));
98+
}
99+
100+
public final C shared(String imageName) {
101+
return shared(imageName, new ArrayList<>());
102+
}
103+
104+
public final C shared(String imageName, List<NamedContainerModifier<C>> namedContainerModifiers) {
105+
final ContainerKey<C> containerKey = new ContainerKey<>(getClass(), DockerImageName.parse(imageName), namedContainerModifiers);
91106
// We deliberately avoid creating the container itself eagerly during the evaluation of the map
92107
// value.
93108
// Container creation can be exceedingly slow.
@@ -102,39 +117,73 @@ public final C shared(String imageName, String... methods) {
102117
* Returns an exclusive instance of the testcontainer.
103118
*/
104119
@SuppressWarnings("unchecked")
120+
@Deprecated
105121
public final C exclusive(String imageName, String... methods) {
106-
return (C) createAndStartContainer(DockerImageName.parse(imageName), Stream.of(methods).toList());
122+
return exclusive(imageName, (NamedContainerModifier<C>) Stream.of(methods).map(n -> new NamedContainerModifierImpl<C>(n, resolveModifierByName(n))).toList());
123+
}
124+
125+
public final C exclusive(String imageName) {
126+
return exclusive(imageName, new ArrayList<>());
127+
}
128+
129+
public final C exclusive(String imageName, NamedContainerModifier<C>... namedContainerModifiers) {
130+
return exclusive(imageName, List.of(namedContainerModifiers));
131+
}
132+
133+
public final C exclusive(String imageName, List<NamedContainerModifier<C>> namedContainerModifiers) {
134+
return (C) createAndStartContainer(DockerImageName.parse(imageName), namedContainerModifiers);
107135
}
108136

109-
private GenericContainer<?> createAndStartContainer(DockerImageName imageName, List<String> methodNames) {
110-
LOGGER.info("Creating new shared container based on {} with {}.", imageName, methodNames);
111-
try {
112-
GenericContainer<?> container = createNewContainer(imageName);
113-
final var methods = new ArrayList<Method>();
114-
for (String methodName : methodNames) {
115-
methods.add(getClass().getMethod(methodName, container.getClass()));
137+
public interface NamedContainerModifier<C extends GenericContainer<?>> {
138+
139+
String name();
140+
Consumer<C> modifier();
141+
}
142+
143+
public record NamedContainerModifierImpl<C extends GenericContainer<?>>(String name, Consumer<C> method) implements NamedContainerModifier<C>{
144+
public String name() {
145+
return name;
146+
}
147+
public Consumer<C> modifier() {
148+
return method;
149+
}
150+
}
151+
152+
private Consumer<C> resolveModifierByName(String methodName) {
153+
final ContainerFactory<C> self = this;
154+
Consumer<C> resolvedMethod = c -> {
155+
try {
156+
Class<? extends GenericContainer> containerClass = c.getClass();
157+
Method method = self.getClass().getMethod(methodName, containerClass);
158+
method.invoke(self, c);
159+
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
160+
throw new RuntimeException(e);
116161
}
117-
final var logConsumer = new Slf4jLogConsumer(LOGGER) {
162+
};
163+
return resolvedMethod;
164+
}
118165

119-
public void accept(OutputFrame frame) {
120-
if (frame.getUtf8StringWithoutLineEnding().trim().length() > 0) {
121-
super.accept(frame);
122-
}
123-
}
166+
private C createAndStartContainer(DockerImageName imageName, List<NamedContainerModifier<C>> namedContainerModifiers) {
167+
LOGGER.info("Creating new shared container based on {} with {}.", imageName, Lists.transform(namedContainerModifiers, c->c.name()));
168+
C container = createNewContainer(imageName);
169+
final var logConsumer = new Slf4jLogConsumer(LOGGER) {
124170

125-
};
126-
getTestContainerLogMdcBuilder(imageName, methodNames).produceMappings(logConsumer::withMdc);
127-
container.withLogConsumer(logConsumer);
128-
for (Method method : methods) {
129-
LOGGER.info("Calling {} in {} on new shared container based on {}.",
130-
method.getName(), getClass().getName(), imageName);
131-
method.invoke(this, container);
171+
public void accept(OutputFrame frame) {
172+
if (frame.getUtf8StringWithoutLineEnding().trim().length() > 0) {
173+
super.accept(frame);
174+
}
132175
}
133-
container.start();
134-
return container;
135-
} catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
136-
throw new RuntimeException(e);
176+
177+
};
178+
getTestContainerLogMdcBuilder(imageName, namedContainerModifiers).produceMappings(logConsumer::withMdc);
179+
container.withLogConsumer(logConsumer);
180+
for (NamedContainerModifier<C> resolvedNamedContainerModifier : namedContainerModifiers) {
181+
LOGGER.info("Calling {} in {} on new shared container based on {}.",
182+
resolvedNamedContainerModifier.name(), getClass().getName(), imageName);
183+
resolvedNamedContainerModifier.modifier().accept(container);
137184
}
185+
container.start();
186+
return container;
138187
}
139188

140189
}

0 commit comments

Comments
 (0)