Kafka Cluster using Containers - p3
Utilities
These are some applications which I find useful in working with the kafka cluster.
All the containers for these apps are running on the fourth server, kfk234.
The compose files can be run from the same directory or each app compose file can have its own directory.
Zoonavigator
kfk234-zoonavigator.yaml
version: '2.4'
name: zoonavigator
services:
zoonavigator:
image: elkozmon/zoonavigator:latest
hostname: kfk234.domain.dom
container_name: zoonavigator
restart: "unless-stopped"
ports:
- "8001:8001"
environment:
HTTP_PORT: 8001
AUTO_CONNECT_CONNECTION_STRING: "kfk231:2181,kfk232:2181,kfk233:2181/kafka"
extra_hosts:
- "kfk231:192.168.122.231"
- "kfk231-kafka:192.168.122.231"
- "kfk231-zk:192.168.122.231"
- "kfk231-cnt:192.168.122.231"
- "kfk231-reg:192.168.122.231"
- "kfk231.domain.dom:192.168.122.231"
- "kfk232:192.168.122.232"
- "kfk232-kafka:192.168.122.232"
- "kfk232-zk:192.168.122.232"
- "kfk232-cnt:192.168.122.232"
- "kfk232-reg:192.168.122.232"
- "kfk232.domain.dom:192.168.122.232"
- "kfk233:192.168.122.233"
- "kfk233-kafka:192.168.122.233"
- "kfk233-zk:192.168.122.233"
- "kfk233-cnt:192.168.122.233"
- "kfk233-reg:192.168.122.233"
- "kfk233.domain.dom:192.168.122.233"
- "kfk234:192.168.122.234"
- "kfk234.domain.dom:192.168.122.234"
docker compose -f kfk234-zoonavigator.yaml up -d
Accesing it: http://192.168.122.234:8001
kafka-ui
kfk234-kafka-ui.yaml
version: '2.4'
name: kafka-ui
services:
# https://github.com/provectus/kafka-ui
kafka-ui:
image: provectuslabs/kafka-ui:v0.5.0
hostname: kfk234.domain.dom
container_name: kafka-ui
restart: "unless-stopped"
ports:
- 8013:8080
environment:
KAFKA_CLUSTERS_0_NAME: kfk-cluster
KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kfk231:9092,kfk232:9092,kfk233:9092
#KAFKA_CLUSTERS_0_METRICS_PORT: 9997
KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://192.168.122.234:8081/
KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: kfk231
KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://kfk231:8083
KAFKA_CLUSTERS_0_KAFKACONNECT_1_NAME: kfk232
KAFKA_CLUSTERS_0_KAFKACONNECT_1_ADDRESS: http://kfk232:8083
KAFKA_CLUSTERS_0_KAFKACONNECT_2_NAME: kfk233
KAFKA_CLUSTERS_0_KAFKACONNECT_2_ADDRESS: http://kfk233:8083
extra_hosts:
- "kfk231:192.168.122.231"
- "kfk231-kafka:192.168.122.231"
- "kfk231-zk:192.168.122.231"
- "kfk231-cnt:192.168.122.231"
- "kfk231-reg:192.168.122.231"
- "kfk231.domain.dom:192.168.122.231"
- "kfk232:192.168.122.232"
- "kfk232-kafka:192.168.122.232"
- "kfk232-zk:192.168.122.232"
- "kfk232-cnt:192.168.122.232"
- "kfk232-reg:192.168.122.232"
- "kfk232.domain.dom:192.168.122.232"
- "kfk233:192.168.122.233"
- "kfk233-kafka:192.168.122.233"
- "kfk233-zk:192.168.122.233"
- "kfk233-cnt:192.168.122.233"
- "kfk233-reg:192.168.122.233"
- "kfk233.domain.dom:192.168.122.233"
- "kfk234:192.168.122.234"
- "kfk234.domain.dom:192.168.122.234"
Kafka manager - CMAK
kfk234-kafka-manager.yaml
version: '2.4'
name: kafka-manager
services:
kafka-manager:
image: hlebalbau/kafka-manager:stable
hostname: kfk234.domain.dom
container_name: kafka-manager
restart: "unless-stopped"
ports:
- "9000:9000"
environment:
ZK_HOSTS: "kfk231:2181,kfk232:2181,kfk233:2181/kafka"
APPLICATION_SECRET: change_me_please
extra_hosts:
- "kfk231:192.168.122.231"
- "kfk231-kafka:192.168.122.231"
- "kfk231-zk:192.168.122.231"
- "kfk231-cnt:192.168.122.231"
- "kfk231-reg:192.168.122.231"
- "kfk231.domain.dom:192.168.122.231"
- "kfk232:192.168.122.232"
- "kfk232-kafka:192.168.122.232"
- "kfk232-zk:192.168.122.232"
- "kfk232-cnt:192.168.122.232"
- "kfk232-reg:192.168.122.232"
- "kfk232.domain.dom:192.168.122.232"
- "kfk233:192.168.122.233"
- "kfk233-kafka:192.168.122.233"
- "kfk233-zk:192.168.122.233"
- "kfk233-cnt:192.168.122.233"
- "kfk233-reg:192.168.122.233"
- "kfk233.domain.dom:192.168.122.233"
- "kfk234:192.168.122.234"
- "kfk234.domain.dom:192.168.122.234"
docker compose -f kfk234-kafka-manager.yaml up -d
Accessing it: http://192.168.122.234:9000
Kafka Connect UI
kfk234-kafka-connect-ui.yaml
version: '2.4'
name: kafka-connect-ui
services:
# https://github.com/Landoop/kafka-connect-ui
kafka-connect-ui:
image: landoop/kafka-connect-ui:0.9.7
hostname: kfk234.domain.dom
container_name: kafka-connect-ui
restart: "unless-stopped"
ports:
- "8005:8000"
environment:
CONNECT_URL: "http://kfk231:8083,http://kfk232:8083,http://kfk233:8083"
extra_hosts:
- "kfk231:192.168.122.231"
- "kfk231-kafka:192.168.122.231"
- "kfk231-zk:192.168.122.231"
- "kfk231-cnt:192.168.122.231"
- "kfk231-reg:192.168.122.231"
- "kfk231.domain.dom:192.168.122.231"
- "kfk232:192.168.122.232"
- "kfk232-kafka:192.168.122.232"
- "kfk232-zk:192.168.122.232"
- "kfk232-cnt:192.168.122.232"
- "kfk232-reg:192.168.122.232"
- "kfk232.domain.dom:192.168.122.232"
- "kfk233:192.168.122.233"
- "kfk233-kafka:192.168.122.233"
- "kfk233-zk:192.168.122.233"
- "kfk233-cnt:192.168.122.233"
- "kfk233-reg:192.168.122.233"
- "kfk233.domain.dom:192.168.122.233"
- "kfk234:192.168.122.234"
- "kfk234.domain.dom:192.168.122.234"
docker compose -f kfk234-kafka-connect-ui.yaml up -d
Accessing it: http://192.168.122.234:8005
Kafka Topics UI
kfk234-topics-ui.yaml
version: '2.4'
name: kafka-topics-ui
services:
# https://github.com/Landoop/kafka-topics-ui
kafka-topics-ui:
image: landoop/kafka-topics-ui:0.9.4
hostname: kfk234.domain.dom
container_name: kafka-topics-ui
restart: "unless-stopped"
ports:
- "8010:8000"
environment:
KAFKA_REST_PROXY_URL: "http://192.168.122.234:8082"
PROXY: "TRUE"
extra_hosts:
- "kfk231:192.168.122.231"
- "kfk231-kafka:192.168.122.231"
- "kfk231-zk:192.168.122.231"
- "kfk231-cnt:192.168.122.231"
- "kfk231-reg:192.168.122.231"
- "kfk231.domain.dom:192.168.122.231"
- "kfk232:192.168.122.232"
- "kfk232-kafka:192.168.122.232"
- "kfk232-zk:192.168.122.232"
- "kfk232-cnt:192.168.122.232"
- "kfk232-reg:192.168.122.232"
- "kfk232.domain.dom:192.168.122.232"
- "kfk233:192.168.122.233"
- "kfk233-kafka:192.168.122.233"
- "kfk233-zk:192.168.122.233"
- "kfk233-cnt:192.168.122.233"
- "kfk233-reg:192.168.122.233"
- "kfk233.domain.dom:192.168.122.233"
- "kfk234:192.168.122.234"
- "kfk234.domain.dom:192.168.122.234"
docker compose -f kfk234-topics-ui.yaml up -d
Accessing it: http://192.168.122.234:8010
Schema Registry UI
kfk234-schema-registry-ui.yaml
version: '3'
name: schema-registry-ui
services:
# A web tool that allows you to create / view / search / evolve / view
# history & configure Avro schemas of your Kafka cluster.
# https://github.com/Landoop/schema-registry-ui
schema-registry-ui:
image: landoop/schema-registry-ui:0.9.4
hostname: kfk234domain.dom
container_name: schema-registry-ui
restart: "unless-stopped"
ports:
- "8011:8000"
environment:
SCHEMAREGISTRY_URL: http://192.168.122.234:8081/
PROXY: "TRUE"
extra_hosts:
- "kfk231:192.168.122.231"
- "kfk231-kafka:192.168.122.231"
- "kfk231-zk:192.168.122.231"
- "kfk231-cnt:192.168.122.231"
- "kfk231-reg:192.168.122.231"
- "kfk231.domain.dom:192.168.122.231"
- "kfk232:192.168.122.232"
- "kfk232-kafka:192.168.122.232"
- "kfk232-zk:192.168.122.232"
- "kfk232-cnt:192.168.122.232"
- "kfk232-reg:192.168.122.232"
- "kfk232.domain.dom:192.168.122.232"
- "kfk233:192.168.122.233"
- "kfk233-kafka:192.168.122.233"
- "kfk233-zk:192.168.122.233"
- "kfk233-cnt:192.168.122.233"
- "kfk233-reg:192.168.122.233"
- "kfk233.domain.dom:192.168.122.233"
- "kfk234:192.168.122.234"
- "kfk234.domain.dom:192.168.122.234"
docker compose -f kfk234-schema-registry-ui.yaml up -d
Accessing it: http://192.168.122.234:8011
AKHQ
kfk234-akhq-without-certificates.yaml
version: '2'
name: akhq
services:
# https://github.com/tchiotludo/akhq/blob/dev/docker-compose.yml
akhq:
image: tchiotludo/akhq
hostname: kfk234.domain.dom
container_name: akhq
restart: "unless-stopped"
ports:
- 8014:8080
environment:
AKHQ_CONFIGURATION: |
akhq:
connections:
docker-kafka-server:
properties:
bootstrap.servers: "kfk231:9092,kfk232:9092,kfk233:9092"
schema-registry:
url: "http://192.168.122.234:8081"
connect:
- name: "connect"
url: "http://kfk231:8083,http://kfk232:8083,http://kfk233:8083"
extra_hosts:
- "kfk231:192.168.122.231"
- "kfk231-kafka:192.168.122.231"
- "kfk231-zk:192.168.122.231"
- "kfk231-cnt:192.168.122.231"
- "kfk231-reg:192.168.122.231"
- "kfk231.domain.dom:192.168.122.231"
- "kfk232:192.168.122.232"
- "kfk232-kafka:192.168.122.232"
- "kfk232-zk:192.168.122.232"
- "kfk232-cnt:192.168.122.232"
- "kfk232-reg:192.168.122.232"
- "kfk232.domain.dom:192.168.122.232"
- "kfk233:192.168.122.233"
- "kfk233-kafka:192.168.122.233"
- "kfk233-zk:192.168.122.233"
- "kfk233-cnt:192.168.122.233"
- "kfk233-reg:192.168.122.233"
- "kfk233.domain.dom:192.168.122.233"
- "kfk234:192.168.122.234"
- "kfk234.domain.dom:192.168.122.234"
With certificates
kfk234-akhq-with-certificates.yaml
version: '2'
name: akhq
services:
# https://github.com/tchiotludo/akhq/blob/dev/docker-compose.yml
akhq:
image: tchiotludo/akhq
hostname: kfk234.domain.dom
container_name: akhq
restart: "unless-stopped"
ports:
- 8014:8080
volumes:
- ./secrets:/etc/kafka/secrets
environment:
AKHQ_CONFIGURATION: |
akhq:
connections:
docker-kafka-server:
properties:
bootstrap.servers: "kfk231:9093,kfk232:9093,kfk233:9093"
security.protocol: SSL
ssl.truststore.location: /etc/kafka/secrets/kafka.kafkaUtil.truststore.jks
ssl.truststore.password: ${SSL_TRUSTSTORE_PASSWD}
ssl.keystore.type: "PKCS12"
ssl.keystore.location: /etc/kafka/secrets/kafka.general.keystore.jks
ssl.keystore.password: ${SSL_KEYSTORE_PASSWD}
ssl.key.password: ${SSL_KEY_PASSWD}
ssl.endpoint.identification.algorithm: ""
schema-registry:
url: "http://192.168.122.234:8081"
connect:
- name: "connect"
url: "http://kfk231:8083,http://kfk232:8083,http://kfk233:8083"
extra_hosts:
- "kfk231:192.168.122.231"
- "kfk231-kafka:192.168.122.231"
- "kfk231-zk:192.168.122.231"
- "kfk231-cnt:192.168.122.231"
- "kfk231-reg:192.168.122.231"
- "kfk231.domain.dom:192.168.122.231"
- "kfk232:192.168.122.232"
- "kfk232-kafka:192.168.122.232"
- "kfk232-zk:192.168.122.232"
- "kfk232-cnt:192.168.122.232"
- "kfk232-reg:192.168.122.232"
- "kfk232.domain.dom:192.168.122.232"
- "kfk233:192.168.122.233"
- "kfk233-kafka:192.168.122.233"
- "kfk233-zk:192.168.122.233"
- "kfk233-cnt:192.168.122.233"
- "kfk233-reg:192.168.122.233"
- "kfk233.domain.dom:192.168.122.233"
- "kfk234:192.168.122.234"
- "kfk234.domain.dom:192.168.122.234"