|
| 1 | +package kafka; |
| 2 | + |
| 3 | +import io.debezium.testing.testcontainers.ConnectorConfiguration; |
| 4 | +import io.debezium.testing.testcontainers.DebeziumContainer; |
| 5 | +import io.questdb.client.Sender; |
| 6 | +import io.questdb.kafka.QuestDBSinkConnector; |
| 7 | +import io.questdb.kafka.QuestDBSinkConnectorConfig; |
| 8 | +import io.questdb.kafka.QuestDBSinkTask; |
| 9 | +import io.questdb.kafka.QuestDBUtils; |
| 10 | +import io.questdb.kafka.JarResolverExtension; |
| 11 | +import org.apache.kafka.clients.consumer.ConsumerConfig; |
| 12 | +import org.apache.kafka.clients.consumer.ConsumerRecord; |
| 13 | +import org.apache.kafka.clients.consumer.KafkaConsumer; |
| 14 | +import org.apache.kafka.common.serialization.StringDeserializer; |
| 15 | +import org.apache.kafka.connect.json.JsonConverter; |
| 16 | +import org.junit.jupiter.api.BeforeAll; |
| 17 | +import org.junit.jupiter.api.Test; |
| 18 | +import org.junit.jupiter.api.extension.RegisterExtension; |
| 19 | +import org.rnorth.ducttape.unreliables.Unreliables; |
| 20 | +import org.slf4j.LoggerFactory; |
| 21 | +import org.testcontainers.containers.GenericContainer; |
| 22 | +import org.testcontainers.containers.KafkaContainer; |
| 23 | +import org.testcontainers.containers.Network; |
| 24 | +import org.testcontainers.containers.PostgreSQLContainer; |
| 25 | +import org.testcontainers.containers.output.Slf4jLogConsumer; |
| 26 | +import org.testcontainers.junit.jupiter.Container; |
| 27 | +import org.testcontainers.junit.jupiter.Testcontainers; |
| 28 | +import org.testcontainers.lifecycle.Startables; |
| 29 | +import org.testcontainers.utility.DockerImageName; |
| 30 | +import org.testcontainers.utility.MountableFile; |
| 31 | + |
| 32 | +import java.sql.Connection; |
| 33 | +import java.sql.DriverManager; |
| 34 | +import java.sql.SQLException; |
| 35 | +import java.sql.Statement; |
| 36 | +import java.sql.Time; |
| 37 | +import java.time.Duration; |
| 38 | +import java.util.ArrayList; |
| 39 | +import java.util.Arrays; |
| 40 | +import java.util.List; |
| 41 | +import java.util.Properties; |
| 42 | +import java.util.UUID; |
| 43 | +import java.util.concurrent.TimeUnit; |
| 44 | +import java.util.stream.Stream; |
| 45 | + |
| 46 | +import static org.hamcrest.MatcherAssert.assertThat; |
| 47 | + |
| 48 | +@Testcontainers |
| 49 | +public class DebeziumIT { |
| 50 | + |
| 51 | + // we need to locate JARs with QuestDB client and Kafka Connect Connector, |
| 52 | + // this is later used to copy to the Kafka Connect container |
| 53 | + @RegisterExtension |
| 54 | + public static JarResolverExtension connectorJarResolver = JarResolverExtension.forClass(QuestDBSinkTask.class); |
| 55 | + @RegisterExtension |
| 56 | + public static JarResolverExtension questdbJarResolver = JarResolverExtension.forClass(Sender.class); |
| 57 | + |
| 58 | + private static Network network = Network.newNetwork(); |
| 59 | + |
| 60 | + private static KafkaContainer kafkaContainer = new KafkaContainer() |
| 61 | + .withNetwork(network); |
| 62 | + |
| 63 | + public static PostgreSQLContainer<?> postgresContainer = |
| 64 | + new PostgreSQLContainer<>(DockerImageName.parse("debezium/postgres:11").asCompatibleSubstituteFor("postgres")) |
| 65 | + .withNetwork(network) |
| 66 | + .withNetworkAliases("postgres"); |
| 67 | + |
| 68 | + public static DebeziumContainer debeziumContainer = |
| 69 | + new DebeziumContainer("debezium/connect:1.9.6.Final") |
| 70 | + .withNetwork(network) |
| 71 | + .withKafka(kafkaContainer) |
| 72 | + .withCopyFileToContainer(MountableFile.forHostPath(connectorJarResolver.getJarPath()), "/kafka/connect/questdb-connector/questdb-connector.jar") |
| 73 | + .withCopyFileToContainer(MountableFile.forHostPath(questdbJarResolver.getJarPath()), "/kafka/connect/questdb-connector/questdb.jar") |
| 74 | + .dependsOn(kafkaContainer) |
| 75 | + .withEnv("CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE", "true") |
| 76 | + .withEnv("CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE", "true"); |
| 77 | + |
| 78 | + @Container |
| 79 | + private static final GenericContainer<?> questDBContainer = new GenericContainer<>("questdb/questdb:6.5.3") |
| 80 | + .withNetwork(network) |
| 81 | + .withExposedPorts(QuestDBUtils.QUESTDB_HTTP_PORT) |
| 82 | + .withLogConsumer(new Slf4jLogConsumer(LoggerFactory.getLogger("questdb"))) |
| 83 | + .withEnv("QDB_CAIRO_COMMIT_LAG", "100") |
| 84 | + .withEnv("JAVA_OPTS", "-Djava.locale.providers=JRE,SPI"); |
| 85 | + |
| 86 | + @BeforeAll |
| 87 | + public static void startContainers() { |
| 88 | + Startables.deepStart(Stream.of( |
| 89 | + kafkaContainer, postgresContainer, debeziumContainer)) |
| 90 | + .join(); |
| 91 | + } |
| 92 | + |
| 93 | + @Test |
| 94 | + public void testSmoke() throws Exception { |
| 95 | + String tableName = "todo"; |
| 96 | + try (Connection connection = getConnection(postgresContainer); |
| 97 | + Statement statement = connection.createStatement()) { |
| 98 | + |
| 99 | + statement.execute("create schema todo"); |
| 100 | + statement.execute("create table todo.Todo (id int8 not null, " + |
| 101 | + "title varchar(255), primary key (id))"); |
| 102 | + statement.execute("alter table todo.Todo replica identity full"); |
| 103 | + statement.execute("insert into todo.Todo values (1, " + |
| 104 | + "'Learn CDC')"); |
| 105 | + statement.execute("insert into todo.Todo values (2, " + |
| 106 | + "'Learn Debezium')"); |
| 107 | + |
| 108 | + ConnectorConfiguration connector = ConnectorConfiguration |
| 109 | + .forJdbcContainer(postgresContainer) |
| 110 | + .with("database.server.name", "dbserver1"); |
| 111 | + |
| 112 | + |
| 113 | + debeziumContainer.registerConnector("my-connector", connector); |
| 114 | + |
| 115 | + ConnectorConfiguration questSink = ConnectorConfiguration.create() |
| 116 | + .with("connector.class", QuestDBSinkConnector.class.getName()) |
| 117 | + .with("host", questDBContainer.getNetworkAliases().get(0)) |
| 118 | + .with("tasks.max", "1") |
| 119 | + .with("topics", "dbserver1.todo.todo") |
| 120 | + .with(QuestDBSinkConnectorConfig.TABLE_CONFIG, tableName) |
| 121 | + .with("key.converter", JsonConverter.class.getName()) |
| 122 | + .with("value.converter", JsonConverter.class.getName()) |
| 123 | + .with("transforms", "unwrap") |
| 124 | + .with("transforms.unwrap.type", "io.debezium.transforms.ExtractNewRecordState") |
| 125 | + .with(QuestDBSinkConnectorConfig.INCLUDE_KEY_CONFIG, "false"); |
| 126 | + |
| 127 | + debeziumContainer.registerConnector("questdb-sink", questSink); |
| 128 | + |
| 129 | + |
| 130 | + QuestDBUtils.assertSqlEventually(questDBContainer, "\"id\",\"title\"\r\n" |
| 131 | + + "1,\"Learn CDC\"\r\n" |
| 132 | + + "2,\"Learn Debezium\"\r\n", "select id, title from " + tableName); |
| 133 | + } |
| 134 | + } |
| 135 | + |
| 136 | + private static Connection getConnection( |
| 137 | + PostgreSQLContainer<?> postgresContainer) |
| 138 | + throws SQLException { |
| 139 | + |
| 140 | + return DriverManager.getConnection(postgresContainer.getJdbcUrl(), |
| 141 | + postgresContainer.getUsername(), |
| 142 | + postgresContainer.getPassword()); |
| 143 | + } |
| 144 | +} |
0 commit comments