add ksqldb

README.md

@@ -280,8 +280,11 @@ A collection of delicious docker recipes.
 - [x] browserless/chrome
 - [x] certbot
 - [x] codercom/code-server
-- [x] confluentinc/cp-kafka-mqtt
+- [x] confluentinc
-- [x] confluentinc/cp-kafka-rest
+  - [x] cp-kafka-mqtt
+  - [x] cp-kafka-rest
+  - [x] ksqldb-cli
+  - [x] ksqldb-server
 - [x] streamsets/datacollector
 - [x] daskdev
   - [x] dask

kafka-rest/docker-compose.yml

@@ -1,16 +1,29 @@
+version: "3.7"
+
+services:
+
+  schema-registry:
+    image: confluentinc/cp-schema-registry:5.4.0
+    container_name: schema-registry
+    hostname: schema-registry
+    environment:
+      - SCHEMA_REGISTRY_HOST_NAME=schema-registry
+      - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181
+    extra_hosts:
+      - zookeeper:10.0.0.21
+    restart: unless-stopped
+
   kafka-rest:
-  image: confluentinc/cp-kafka-rest
+    image: confluentinc/cp-kafka-rest:5.4.0
+    container_name: kafka-rest
+    hostname: kafka-rest
     ports:
       - "8082:8082"
     environment:
       - KAFKA_REST_HOST_NAME=kafka-rest
-    - KAFKA_REST_ZOOKEEPER_CONNECT=zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
+      - KAFKA_REST_ZOOKEEPER_CONNECT=zookeeper:2181
-    - KAFKA_REST_BOOTSTRAP_SERVERS=kafka1:9092,kafka2:9092,kafka3:9092
+      - KAFKA_REST_BOOTSTRAP_SERVERS=kafka:9092
     extra_hosts:
-    - zookeeper1:10.0.0.21
+      - zookeeper:10.0.0.21
-    - zookeeper2:10.0.0.22
+      - kafka:10.0.0.21
-    - zookeeper3:10.0.0.23
-    - kafka1:10.0.0.21
-    - kafka2:10.0.0.22
-    - kafka3:10.0.0.23
     restart: unless-stopped

ksqldb/README.md

@@ -0,0 +1,28 @@
+ksqldb
+======
+
+[ksqlDB][1] is an event streaming database.
+
+## Up and Running
+
+```bash
+$ docker-compose up -d
+$ curl http://127.0.0.1:8088/info
+$ docker-compose exec ksqldb-cli ksql http://ksql-server:8088
+>>> CREATE TABLE products (product_name VARCHAR, cost DOUBLE)
+    WITH (kafka_topic='products', partitions=3, value_format='json');
+>>> CREATE STREAM products_changelog AS
+    SELECT * FROM products EMIT CHANGES;
+>>> SHOW TOPICS;
+>>> SHOW TABLES;
+>>> SHOW STREAMS;
+```
+
+## Docs
+
+- ksql: https://docs.confluent.io/current/ksql/docs/index.html
+- ksql-server: https://docs.confluent.io/current/ksql/docs/installation/server-config/index.html
+- ksql-cli: https://docs.confluent.io/current/ksql/docs/installation/cli-config.html
+- connector: https://docs.ksqldb.io/en/latest/concepts/connectors/
+
+[1]: https://ksqldb.io/

ksqldb/data/connect.properties

@@ -0,0 +1,12 @@
+bootstrap.servers=kafka:9092
+plugin.path=/data/plugins
+group.id=ksql-connect-cluster
+key.converter=org.apache.kafka.connect.json.JsonConverter
+value.converter=org.apache.kafka.connect.json.JsonConverter
+value.converter.schemas.enable=false
+config.storage.topic=ksql-connect-configs
+offset.storage.topic=ksql-connect-offsets
+status.storage.topic=ksql-connect-statuses
+config.storage.replication.factor=1
+offset.storage.replication.factor=1
+status.storage.replication.factor=1

ksqldb/data/plugins/README.md

@@ -0,0 +1,3 @@
+- https://www.confluent.io/hub/confluentinc/kafka-connect-jdbc
+- https://www.confluent.io/hub/mongodb/kafka-connect-mongodb
+- https://www.confluent.io/hub/neo4j/kafka-connect-neo4j

ksqldb/docker-compose.yml

@@ -0,0 +1,33 @@
+version: "3.7"
+
+services:
+
+  ksqldb-server:
+    image: confluentinc/ksqldb-server:0.6.0
+    ports:
+      - "8088:8088"
+    environment:
+      - KSQL_LISTENERS=http://0.0.0.0:8088
+      - KSQL_BOOTSTRAP_SERVERS=kafka:9092
+      - KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE=true
+      - KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE=true
+      - KSQL_KSQL_CONNECT_WORKER_CONFIG=/data/connect.properties
+      # KSQL_KSQL_CONNECT_URL=kafka-connect:8083
+      # KSQL_KSQL_SCHEMA_REGISTRY_URL=http://schema-registry:8081
+    volumes:
+      - ./data:/data
+    restart: unless-stopped
+
+  ksqldb-cli:
+    image: confluentinc/ksqldb-cli:0.6.0
+    entrypoint: /bin/sh
+    tty: true
+    depends_on:
+      - ksqldb-server
+
+  kafkacat:
+    image: confluentinc/cp-kafkacat:5.4.0
+    entrypoint: /bin/sh
+    tty: true
+    depends_on:
+      - kafka