Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(#3483): Use Kraft-based Kafka as default for internal messaging #3484

Open
wants to merge 3 commits into
base: dev
Choose a base branch
from
Open

feat(#3483): Use Kraft-based Kafka as default for internal messaging #3484

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2a5f0bd
feat(#3483): Use Kraft-based Kafka as default for internal messaging
dominikriemer Feb 14, 2025
71776d9
Let Kafka consumer wait until rebalancing is finished
dominikriemer Feb 15, 2025
cf41d5c
Update helm charts
dominikriemer Feb 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 14 additions & 22 deletions docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,33 +84,26 @@ services:
spnet:

kafka:
image: fogsyio/kafka:2.2.0
image: bitnami/kafka:3.9.0
hostname: kafka
depends_on:
- zookeeper
environment:
# see: https://github.com/confluentinc/schema-registry/issues/648
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092
KAFKA_LISTENERS: PLAINTEXT://:9092
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_ADVERTISED_HOST_NAME: kafka
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
- KAFKA_CFG_NODE_ID=0
- KAFKA_CFG_PROCESS_ROLES=controller,broker
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
- KAFKA_CFG_MESSAGE_MAX_BYTES=5000012
- KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES=5000012
- KAFKA_CFG_REPLICA_FETCH_MAX_BYTES=10000000
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
volumes:
- kafka:/kafka
- /var/run/docker.sock:/var/run/docker.sock
- kafka3:/bitnami
logging: *default-logging
networks:
spnet:

zookeeper:
image: fogsyio/zookeeper:3.4.13
logging: *default-logging
volumes:
- zookeeper:/opt/zookeeper-3.4.13
networks:
spnet:

influxdb:
image: influxdb:2.6
environment:
Expand Down Expand Up @@ -144,10 +137,9 @@ services:
spnet:

volumes:
kafka:
kafka3:
files:
couchdb:
zookeeper:
influxdb:
influxdb2:
backend:
Expand Down
26 changes: 14 additions & 12 deletions installer/cli/deploy/standalone/kafka/docker-compose.dev.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,18 @@ services:
kafka:
ports:
- "9094:9094"
depends_on:
- zookeeper
environment:
# see: https://github.com/confluentinc/schema-registry/issues/648
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9094 # Replace localhost with your external address if Kafka should be reachable from external systems.
KAFKA_LISTENERS: PLAINTEXT://:9092,OUTSIDE://:9094
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_ADVERTISED_HOST_NAME: kafka
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_MESSAGE_MAX_BYTES: 5000012
KAFKA_FETCH_MESSAGE_MAX_BYTES: 5000012
KAFKA_REPLICA_FETCH_MAX_BYTES: 10000000
# KRaft settings
- KAFKA_CFG_NODE_ID=0
- KAFKA_CFG_PROCESS_ROLES=controller,broker
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
# Listeners
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
- KAFKA_CFG_MESSAGE_MAX_BYTES=5000012
- KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES=5000012
- KAFKA_CFG_REPLICA_FETCH_MAX_BYTES=10000000
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER

31 changes: 15 additions & 16 deletions installer/cli/deploy/standalone/kafka/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,24 +15,23 @@

services:
kafka:
image: fogsyio/kafka:2.2.0
image: bitnami/kafka:3.9.0
hostname: kafka
depends_on:
- zookeeper
environment:
# see: https://github.com/confluentinc/schema-registry/issues/648
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092
KAFKA_LISTENERS: PLAINTEXT://:9092
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_ADVERTISED_HOST_NAME: kafka
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_MESSAGE_MAX_BYTES: 5000012
KAFKA_FETCH_MESSAGE_MAX_BYTES: 5000012
KAFKA_REPLICA_FETCH_MAX_BYTES: 10000000
- KAFKA_CFG_NODE_ID=0
- KAFKA_CFG_PROCESS_ROLES=controller,broker
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
# Listeners
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
- KAFKA_CFG_MESSAGE_MAX_BYTES=5000012
- KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES=5000012
- KAFKA_CFG_REPLICA_FETCH_MAX_BYTES=10000000
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
volumes:
- kafka:/kafka
- /var/run/docker.sock:/var/run/docker.sock
- kafka3:/bitnami
logging:
driver: "json-file"
options:
Expand All @@ -42,7 +41,7 @@ services:
spnet:

volumes:
kafka:
kafka3:

networks:
spnet:
Expand Down
41 changes: 14 additions & 27 deletions installer/compose/docker-compose.full.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,32 +65,22 @@ services:
spnet:

kafka:
image: fogsyio/kafka:2.2.0
image: bitnami/kafka:3.9.0
hostname: kafka
depends_on:
- zookeeper
environment:
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9094 # Replace localhost with your external address if Kafka should be reachable from external systems.
KAFKA_LISTENERS: PLAINTEXT://:9092,OUTSIDE://:9094
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_ADVERTISED_HOST_NAME: kafka
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_MESSAGE_MAX_BYTES: 5000012
KAFKA_FETCH_MESSAGE_MAX_BYTES: 5000012
KAFKA_REPLICA_FETCH_MAX_BYTES: 10000000
- KAFKA_CFG_NODE_ID=0
- KAFKA_CFG_PROCESS_ROLES=controller,broker
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
- KAFKA_CFG_MESSAGE_MAX_BYTES=5000012
- KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES=5000012
- KAFKA_CFG_REPLICA_FETCH_MAX_BYTES=10000000
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
volumes:
- kafka:/kafka
- /var/run/docker.sock:/var/run/docker.sock
logging: *default-logging
restart: unless-stopped
networks:
spnet:

zookeeper:
image: fogsyio/zookeeper:3.4.13
volumes:
- zookeeper:/opt/zookeeper-3.4.13
- kafka3:/bitnami
logging: *default-logging
restart: unless-stopped
networks:
Expand Down Expand Up @@ -155,14 +145,11 @@ volumes:
backend:
connect:
couchdb:
kafka:
zookeeper:
kafka3:
influxdb:
influxdb2:
files:
nginx:



networks:
spnet:
Expand Down
39 changes: 14 additions & 25 deletions installer/compose/docker-compose.quickstart.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,32 +65,22 @@ services:
spnet:

kafka:
image: fogsyio/kafka:2.2.0
image: bitnami/kafka:3.9.0
hostname: kafka
depends_on:
- zookeeper
environment:
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9094 # Replace localhost with your external address if Kafka should be reachable from external systems.
KAFKA_LISTENERS: PLAINTEXT://:9092,OUTSIDE://:9094
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_ADVERTISED_HOST_NAME: kafka
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_MESSAGE_MAX_BYTES: 5000012
KAFKA_FETCH_MESSAGE_MAX_BYTES: 5000012
KAFKA_REPLICA_FETCH_MAX_BYTES: 10000000
volumes:
- kafka:/kafka
- /var/run/docker.sock:/var/run/docker.sock
logging: *default-logging
restart: unless-stopped
networks:
spnet:

zookeeper:
image: fogsyio/zookeeper:3.4.13
- KAFKA_CFG_NODE_ID=0
- KAFKA_CFG_PROCESS_ROLES=controller,broker
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
- KAFKA_CFG_MESSAGE_MAX_BYTES=5000012
- KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES=5000012
- KAFKA_CFG_REPLICA_FETCH_MAX_BYTES=10000000
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
volumes:
- zookeeper:/opt/zookeeper-3.4.13
- kafka3:/bitnami
logging: *default-logging
restart: unless-stopped
networks:
Expand Down Expand Up @@ -141,8 +131,7 @@ volumes:
backend:
connect:
couchdb:
kafka:
zookeeper:
kafka3:
influxdb:
influxdb2:
files:
Expand Down
39 changes: 14 additions & 25 deletions installer/compose/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,32 +65,22 @@ services:
spnet:

kafka:
image: fogsyio/kafka:2.2.0
image: bitnami/kafka:3.9.0
hostname: kafka
depends_on:
- zookeeper
environment:
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9094 # Replace localhost with your external address if Kafka should be reachable from external systems.
KAFKA_LISTENERS: PLAINTEXT://:9092,OUTSIDE://:9094
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_ADVERTISED_HOST_NAME: kafka
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_MESSAGE_MAX_BYTES: 5000012
KAFKA_FETCH_MESSAGE_MAX_BYTES: 5000012
KAFKA_REPLICA_FETCH_MAX_BYTES: 10000000
volumes:
- kafka:/kafka
- /var/run/docker.sock:/var/run/docker.sock
logging: *default-logging
restart: unless-stopped
networks:
spnet:

zookeeper:
image: fogsyio/zookeeper:3.4.13
- KAFKA_CFG_NODE_ID=0
- KAFKA_CFG_PROCESS_ROLES=controller,broker
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
- KAFKA_CFG_MESSAGE_MAX_BYTES=5000012
- KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES=5000012
- KAFKA_CFG_REPLICA_FETCH_MAX_BYTES=10000000
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
volumes:
- zookeeper:/opt/zookeeper-3.4.13
- kafka3:/bitnami
logging: *default-logging
restart: unless-stopped
networks:
Expand Down Expand Up @@ -131,8 +121,7 @@ volumes:
backend:
connect:
couchdb:
kafka:
zookeeper:
kafka3:
influxdb:
influxdb2:
files:
Expand Down
16 changes: 1 addition & 15 deletions installer/k8s/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ rm -rf ${HOME}/streampipes-k8s
| Parameter Name | Description | Value |
|---------------------------------------------|------------------------------------------------------------------------------------------|-------------|
| external.kafka.appName | Kafka application name | "kafka" |
| external.kafka.version | Kafka version | 2.2.0 |
| external.kafka.version | Kafka version | 3.9.0 |
| external.kafka.port | Port for the Kafka service | 9092 |
| external.kafka.external.hostname | Name which will be advertised to external clients. Clients which use (default) port 9094 | "localhost" |
| external.kafka.service.name | Name of the Kafka service | "kafka" |
Expand All @@ -237,20 +237,6 @@ rm -rf ${HOME}/streampipes-k8s
| external.kafka.persistence.pvName | Name of the Kafka PersistentVolume | "kafka-pv" |
|

#### Zookeeper common parameters

| Parameter Name | Description | Value |
|-------------------------------------------------|---------------------------------------------|-----------------|
| external.zookeeper.appName | ZooKeeper application name | "zookeeper" |
| external.zookeeper.version | ZooKeeper version | 3.4.13 |
| external.zookeeper.port | Port for the ZooKeeper service | 2181 |
| external.zookeeper.service.name | Name of the ZooKeeper service | "zookeeper" |
| external.zookeeper.service.port | TargetPort of the ZooKeeper service | 2181 |
| external.zookeeper.persistence.storageClassName | Storage class name for ZooKeeper PVs | "hostpath" |
| external.zookeeper.persistence.storageSize | Size of the ZooKeeper PV | "1Gi" |
| external.zookeeper.persistence.claimName | Name of the ZooKeeper PersistentVolumeClaim | "zookeeper-pvc" |
| external.zookeeper.persistence.pvName | Name of the ZooKeeper PersistentVolume | "zookeeper-pv" |

#### Pulsar common parameters

| Parameter Name | Description | Value |
Expand Down
7 changes: 0 additions & 7 deletions installer/k8s/templates/core/backend-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,6 @@ spec:
value: "{{ .Values.external.kafka.service.name }}"
- name: SP_KAFKA_PORT
value: "{{ .Values.external.kafka.service.port }}"
- name: SP_ZOOKEEPER_HOST
value: "{{ .Values.external.zookeeper.service.name }}"
- name: SP_ZOOKEEPER_PORT
value: "{{ .Values.external.zookeeper.service.port }}"
{{- end }}
{{- if eq .Values.preferredBroker "nats" }}
value: "nats"
- name: SP_NATS_HOST
value: "{{ .Values.external.nats.service.name }}"
- name: SP_NATS_PORT
Expand Down
Loading
Loading