Skip to content

Commit bef59b0

Browse files
pedro93pedro93
authoredJan 24, 2023
feat(quickstart): Remove kafka-setup as a hard deployment requirement (datahub-project#7073)
1 parent db96849 commit bef59b0

28 files changed

+706
-489
lines changed
 

‎.github/workflows/build-and-test.yml

+4
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@ jobs:
7373
- uses: actions/setup-python@v4
7474
with:
7575
python-version: "3.7"
76+
- name: Download YQ
77+
uses: chrisdickinson/setup-yq@v1.0.1
78+
with:
79+
yq-version: v4.28.2
7680
- name: Quickstart Compose Validation
7781
run: ./docker/quickstart/generate_and_compare.sh
7882

‎docker/datahub-mce-consumer/env/docker.env

+5-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ MCE_CONSUMER_ENABLED=true
22
EBEAN_DATASOURCE_USERNAME=datahub
33
EBEAN_DATASOURCE_PASSWORD=datahub
44
EBEAN_DATASOURCE_HOST=mysql:3306
5-
EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
5+
EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
66
EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
77
KAFKA_BOOTSTRAP_SERVER=broker:29092
88
KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
@@ -15,6 +15,10 @@ JAVA_OPTS=-Xms1g -Xmx1g
1515
ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml
1616
DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
1717
DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
18+
ENTITY_SERVICE_ENABLE_RETENTION=true
19+
MAE_CONSUMER_ENABLED=false
20+
PE_CONSUMER_ENABLED=false
21+
UI_INGESTION_ENABLED=false
1822

1923
# Uncomment to configure kafka topic names
2024
# Make sure these names are consistent across the whole deployment

‎docker/docker-compose-with-cassandra.yml

+3-16
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
version: '3.8'
88
services:
99
zookeeper:
10-
image: confluentinc/cp-zookeeper:5.4.0
10+
image: confluentinc/cp-zookeeper:7.2.2
1111
env_file: zookeeper/env/docker.env
1212
hostname: zookeeper
1313
container_name: zookeeper
@@ -17,7 +17,7 @@ services:
1717
- zkdata:/var/lib/zookeeper
1818

1919
broker:
20-
image: confluentinc/cp-kafka:5.4.0
20+
image: confluentinc/cp-kafka:7.2.2
2121
env_file: broker/env/docker.env
2222
hostname: broker
2323
container_name: broker
@@ -29,25 +29,12 @@ services:
2929
volumes:
3030
- broker:/var/lib/kafka/data/
3131

32-
# This "container" is a workaround to pre-create topics
33-
kafka-setup:
34-
build:
35-
context: kafka-setup
36-
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
37-
env_file: kafka-setup/env/docker.env
38-
hostname: kafka-setup
39-
container_name: kafka-setup
40-
depends_on:
41-
- broker
42-
- schema-registry
43-
4432
schema-registry:
45-
image: confluentinc/cp-schema-registry:5.4.0
33+
image: confluentinc/cp-schema-registry:7.2.2
4634
env_file: schema-registry/env/docker.env
4735
hostname: schema-registry
4836
container_name: schema-registry
4937
depends_on:
50-
- zookeeper
5138
- broker
5239
ports:
5340
- "8081:8081"

‎docker/docker-compose-without-neo4j.m1.yml

-6
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,3 @@
11
services:
2-
broker:
3-
image: kymeric/cp-kafka:latest
42
mysql:
53
image: mariadb:10.5.8
6-
schema-registry:
7-
image: eugenetea/schema-registry-arm64:latest
8-
zookeeper:
9-
image: kymeric/cp-zookeeper:latest

‎docker/docker-compose-without-neo4j.override.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ services:
66
hostname: mysql
77
image: mysql:5.7
88
env_file: mysql/env/docker.env
9-
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
9+
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
1010
ports:
1111
- ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
1212
volumes:

‎docker/docker-compose-without-neo4j.yml

+3-16
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
version: '3.8'
88
services:
99
zookeeper:
10-
image: confluentinc/cp-zookeeper:5.4.0
10+
image: confluentinc/cp-zookeeper:7.2.2
1111
env_file: zookeeper/env/docker.env
1212
hostname: zookeeper
1313
container_name: zookeeper
@@ -17,7 +17,7 @@ services:
1717
- zkdata:/var/lib/zookeeper
1818

1919
broker:
20-
image: confluentinc/cp-kafka:5.4.0
20+
image: confluentinc/cp-kafka:7.2.2
2121
env_file: broker/env/docker.env
2222
hostname: broker
2323
container_name: broker
@@ -26,25 +26,12 @@ services:
2626
ports:
2727
- ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
2828

29-
# This "container" is a workaround to pre-create topics
30-
kafka-setup:
31-
build:
32-
context: kafka-setup
33-
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
34-
env_file: kafka-setup/env/docker.env
35-
hostname: kafka-setup
36-
container_name: kafka-setup
37-
depends_on:
38-
- broker
39-
- schema-registry
40-
4129
schema-registry:
42-
image: confluentinc/cp-schema-registry:5.4.0
30+
image: confluentinc/cp-schema-registry:7.2.2
4331
env_file: schema-registry/env/docker.env
4432
hostname: schema-registry
4533
container_name: schema-registry
4634
depends_on:
47-
- zookeeper
4835
- broker
4936
ports:
5037
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081

‎docker/docker-compose.consumers-without-neo4j.yml

+3
Original file line numberDiff line numberDiff line change
@@ -25,5 +25,8 @@ services:
2525
env_file: datahub-mce-consumer/env/docker.env
2626
hostname: datahub-mce-consumer
2727
container_name: datahub-mce-consumer
28+
environment:
29+
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
30+
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
2831
ports:
2932
- "9090:9090"

‎docker/docker-compose.consumers.yml

+10
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,15 @@ services:
2727
env_file: datahub-mce-consumer/env/docker.env
2828
hostname: datahub-mce-consumer
2929
container_name: datahub-mce-consumer
30+
environment:
31+
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
32+
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
33+
- NEO4J_HOST=http://neo4j:7474
34+
- NEO4J_URI=bolt://neo4j
35+
- NEO4J_USERNAME=neo4j
36+
- NEO4J_PASSWORD=datahub
37+
- GRAPH_SERVICE_IMPL=neo4j
3038
ports:
3139
- "9090:9090"
40+
depends_on:
41+
- neo4j

‎docker/docker-compose.dev.yml

-11
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,6 @@ services:
2424
- ./elasticsearch-setup/create-indices.sh:/create-indices.sh
2525
- ../metadata-service/restli-servlet-impl/src/main/resources/index/:/index
2626

27-
kafka-setup:
28-
image: linkedin/datahub-kafka-setup:debug
29-
build:
30-
context: ../
31-
dockerfile: ./docker/kafka-setup/Dockerfile
32-
args:
33-
APP_ENV: dev
34-
depends_on:
35-
- broker
36-
- schema-registry
37-
3827
datahub-gms:
3928
image: linkedin/datahub-gms:debug
4029
build:

‎docker/docker-compose.kafka-setup.yml

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Service definitions for Kafka Setup container.
2+
version: '3.8'
3+
services:
4+
5+
# This "container" is a workaround to pre-create topics
6+
kafka-setup:
7+
build:
8+
context: kafka-setup
9+
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
10+
env_file: kafka-setup/env/docker.env
11+
hostname: kafka-setup
12+
container_name: kafka-setup
13+
depends_on:
14+
- broker
15+
- schema-registry

‎docker/docker-compose.m1.yml

-6
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,5 @@
11
services:
2-
broker:
3-
image: kymeric/cp-kafka:latest
42
mysql:
53
image: mariadb:10.5.8
6-
schema-registry:
7-
image: eugenetea/schema-registry-arm64:latest
8-
zookeeper:
9-
image: kymeric/cp-zookeeper:latest
104
neo4j:
115
image: neo4j/neo4j-arm64-experimental:4.0.6-arm64

‎docker/docker-compose.override.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ services:
77
hostname: mysql
88
image: mysql:5.7
99
env_file: mysql/env/docker.env
10-
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
10+
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
1111
ports:
1212
- ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
1313
volumes:

‎docker/docker-compose.tools.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
version: '3.8'
44
services:
55
kafka-rest-proxy:
6-
image: confluentinc/cp-kafka-rest:5.4.0
6+
image: confluentinc/cp-kafka-rest:7.2.2
77
env_file: kafka-rest-proxy/env/docker.env
88
hostname: kafka-rest-proxy
99
container_name: kafka-rest-proxy

‎docker/docker-compose.yml

+3-17
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
version: '3.8'
88
services:
99
zookeeper:
10-
image: confluentinc/cp-zookeeper:5.4.0
10+
image: confluentinc/cp-zookeeper:7.2.2
1111
env_file: zookeeper/env/docker.env
1212
hostname: zookeeper
1313
container_name: zookeeper
@@ -17,7 +17,7 @@ services:
1717
- zkdata:/var/lib/zookeeper
1818

1919
broker:
20-
image: confluentinc/cp-kafka:5.4.0
20+
image: confluentinc/cp-kafka:7.2.2
2121
env_file: broker/env/docker.env
2222
hostname: broker
2323
container_name: broker
@@ -28,26 +28,12 @@ services:
2828
volumes:
2929
- broker:/var/lib/kafka/data/
3030

31-
# This "container" is a workaround to pre-create topics
32-
kafka-setup:
33-
build:
34-
dockerfile: ./docker/kafka-setup/Dockerfile
35-
context: ../
36-
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
37-
env_file: kafka-setup/env/docker.env
38-
hostname: kafka-setup
39-
container_name: kafka-setup
40-
depends_on:
41-
- broker
42-
- schema-registry
43-
4431
schema-registry:
45-
image: confluentinc/cp-schema-registry:5.4.0
32+
image: confluentinc/cp-schema-registry:7.2.2
4633
env_file: schema-registry/env/docker.env
4734
hostname: schema-registry
4835
container_name: schema-registry
4936
depends_on:
50-
- zookeeper
5137
- broker
5238
ports:
5339
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081

‎docker/quickstart/docker-compose-m1.quickstart.yml

+204
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
networks:
2+
default:
3+
name: datahub_network
4+
services:
5+
broker:
6+
container_name: broker
7+
depends_on:
8+
- zookeeper
9+
environment:
10+
- KAFKA_BROKER_ID=1
11+
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
12+
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
13+
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
14+
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
15+
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
16+
- KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
17+
- KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
18+
hostname: broker
19+
image: confluentinc/cp-kafka:7.2.2
20+
ports:
21+
- ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
22+
volumes:
23+
- broker:/var/lib/kafka/data/
24+
datahub-actions:
25+
depends_on:
26+
- datahub-gms
27+
environment:
28+
- DATAHUB_GMS_PROTOCOL=http
29+
- DATAHUB_GMS_HOST=datahub-gms
30+
- DATAHUB_GMS_PORT=8080
31+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
32+
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
33+
- METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
34+
- METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
35+
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
36+
- DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
37+
- KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
38+
hostname: actions
39+
image: acryldata/datahub-actions:${ACTIONS_VERSION:-head}
40+
restart: on-failure:5
41+
datahub-frontend-react:
42+
container_name: datahub-frontend-react
43+
depends_on:
44+
- datahub-gms
45+
environment:
46+
- DATAHUB_GMS_HOST=datahub-gms
47+
- DATAHUB_GMS_PORT=8080
48+
- DATAHUB_SECRET=YouKnowNothing
49+
- DATAHUB_APP_VERSION=1.0
50+
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
51+
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf
52+
-Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml
53+
-Dlogback.debug=false -Dpidfile.path=/dev/null
54+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
55+
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
56+
- ELASTIC_CLIENT_HOST=elasticsearch
57+
- ELASTIC_CLIENT_PORT=9200
58+
hostname: datahub-frontend-react
59+
image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head}
60+
ports:
61+
- ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
62+
volumes:
63+
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
64+
datahub-gms:
65+
container_name: datahub-gms
66+
depends_on:
67+
- neo4j
68+
- mysql
69+
environment:
70+
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
71+
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
72+
- EBEAN_DATASOURCE_USERNAME=datahub
73+
- EBEAN_DATASOURCE_PASSWORD=datahub
74+
- EBEAN_DATASOURCE_HOST=mysql:3306
75+
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
76+
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
77+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
78+
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
79+
- ELASTICSEARCH_HOST=elasticsearch
80+
- ELASTICSEARCH_PORT=9200
81+
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
82+
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
83+
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
84+
- NEO4J_HOST=http://neo4j:7474
85+
- NEO4J_URI=bolt://neo4j
86+
- NEO4J_USERNAME=neo4j
87+
- NEO4J_PASSWORD=datahub
88+
- JAVA_OPTS=-Xms1g -Xmx1g
89+
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
90+
- GRAPH_SERVICE_IMPL=neo4j
91+
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
92+
- ENTITY_SERVICE_ENABLE_RETENTION=true
93+
- MAE_CONSUMER_ENABLED=true
94+
- MCE_CONSUMER_ENABLED=true
95+
- PE_CONSUMER_ENABLED=true
96+
- UI_INGESTION_ENABLED=true
97+
- METADATA_SERVICE_AUTH_ENABLED=false
98+
hostname: datahub-gms
99+
image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head}
100+
ports:
101+
- ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
102+
volumes:
103+
- ${HOME}/.datahub/plugins/:/etc/datahub/plugins
104+
- ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources
105+
elasticsearch:
106+
container_name: elasticsearch
107+
environment:
108+
- discovery.type=single-node
109+
- xpack.security.enabled=false
110+
- ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true
111+
healthcheck:
112+
retries: 4
113+
start_period: 2m
114+
test:
115+
- CMD-SHELL
116+
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s'
117+
|| exit 1
118+
hostname: elasticsearch
119+
image: elasticsearch:7.9.3
120+
mem_limit: 1g
121+
ports:
122+
- ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200
123+
volumes:
124+
- esdata:/usr/share/elasticsearch/data
125+
elasticsearch-setup:
126+
container_name: elasticsearch-setup
127+
depends_on:
128+
- elasticsearch
129+
environment:
130+
- ELASTICSEARCH_HOST=elasticsearch
131+
- ELASTICSEARCH_PORT=9200
132+
- ELASTICSEARCH_PROTOCOL=http
133+
hostname: elasticsearch-setup
134+
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head}
135+
mysql:
136+
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
137+
container_name: mysql
138+
environment:
139+
- MYSQL_DATABASE=datahub
140+
- MYSQL_USER=datahub
141+
- MYSQL_PASSWORD=datahub
142+
- MYSQL_ROOT_PASSWORD=datahub
143+
hostname: mysql
144+
image: mariadb:10.5.8
145+
ports:
146+
- ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
147+
volumes:
148+
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
149+
- mysqldata:/var/lib/mysql
150+
mysql-setup:
151+
container_name: mysql-setup
152+
depends_on:
153+
- mysql
154+
environment:
155+
- MYSQL_HOST=mysql
156+
- MYSQL_PORT=3306
157+
- MYSQL_USERNAME=datahub
158+
- MYSQL_PASSWORD=datahub
159+
- DATAHUB_DB_NAME=datahub
160+
hostname: mysql-setup
161+
image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head}
162+
neo4j:
163+
container_name: neo4j
164+
environment:
165+
- NEO4J_AUTH=neo4j/datahub
166+
- NEO4J_dbms_default__database=graph.db
167+
- NEO4J_dbms_allow__upgrade=true
168+
hostname: neo4j
169+
image: neo4j/neo4j-arm64-experimental:4.0.6-arm64
170+
ports:
171+
- ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474
172+
- ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687
173+
volumes:
174+
- neo4jdata:/data
175+
schema-registry:
176+
container_name: schema-registry
177+
depends_on:
178+
- broker
179+
environment:
180+
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry
181+
- SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT
182+
- SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092
183+
hostname: schema-registry
184+
image: confluentinc/cp-schema-registry:7.2.2
185+
ports:
186+
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
187+
zookeeper:
188+
container_name: zookeeper
189+
environment:
190+
- ZOOKEEPER_CLIENT_PORT=2181
191+
- ZOOKEEPER_TICK_TIME=2000
192+
hostname: zookeeper
193+
image: confluentinc/cp-zookeeper:7.2.2
194+
ports:
195+
- ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
196+
volumes:
197+
- zkdata:/var/lib/zookeeper
198+
version: '2.3'
199+
volumes:
200+
broker: null
201+
esdata: null
202+
mysqldata: null
203+
neo4jdata: null
204+
zkdata: null

‎docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml

+30-49
Original file line numberDiff line numberDiff line change
@@ -21,28 +21,18 @@ services:
2121
- ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
2222
datahub-actions:
2323
depends_on:
24-
- datahub-gms
24+
- datahub-gms
2525
environment:
26-
- DATAHUB_GMS_PROTOCOL=http
27-
- DATAHUB_GMS_HOST=datahub-gms
28-
- DATAHUB_GMS_PORT=8080
29-
- KAFKA_BOOTSTRAP_SERVER=broker:29092
30-
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
31-
- METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
32-
- METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
33-
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
34-
- DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
35-
- KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
36-
- DATAHUB_ACTIONS_SLACK_ENABLED
37-
- DATAHUB_ACTIONS_SLACK_DATAHUB_BASE_URL
38-
- DATAHUB_ACTIONS_SLACK_BOT_TOKEN
39-
- DATAHUB_ACTIONS_SLACK_SIGNING_SECRET
40-
- DATAHUB_ACTIONS_SLACK_CHANNEL
41-
- DATAHUB_ACTIONS_SLACK_SUPPRESS_SYSTEM_ACTIVITY
42-
- DATAHUB_ACTIONS_TEAMS_ENABLED
43-
- DATAHUB_ACTIONS_TEAMS_DATAHUB_BASE_URL
44-
- DATAHUB_ACTIONS_TEAMS_WEBHOOK_URL
45-
- DATAHUB_ACTIONS_TEAMS_SUPPRESS_SYSTEM_ACTIVITY
26+
- DATAHUB_GMS_PROTOCOL=http
27+
- DATAHUB_GMS_HOST=datahub-gms
28+
- DATAHUB_GMS_PORT=8080
29+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
30+
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
31+
- METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
32+
- METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
33+
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
34+
- DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
35+
- KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
4636
hostname: actions
4737
image: acryldata/datahub-actions:${ACTIONS_VERSION:-head}
4838
restart: on-failure:5
@@ -68,31 +58,33 @@ services:
6858
ports:
6959
- ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
7060
volumes:
71-
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
61+
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
7262
datahub-gms:
7363
container_name: datahub-gms
7464
depends_on:
7565
- mysql
7666
environment:
77-
- EBEAN_DATASOURCE_USERNAME=datahub
78-
- EBEAN_DATASOURCE_PASSWORD=datahub
79-
- EBEAN_DATASOURCE_HOST=mysql:3306
80-
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
81-
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
82-
- KAFKA_BOOTSTRAP_SERVER=broker:29092
83-
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
8467
- ELASTICSEARCH_HOST=elasticsearch
85-
- ELASTICSEARCH_PORT=9200
86-
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
87-
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
8868
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
89-
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
90-
- GRAPH_SERVICE_IMPL=elasticsearch
69+
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
70+
- EBEAN_DATASOURCE_USERNAME=datahub
71+
- MCE_CONSUMER_ENABLED=true
72+
- EBEAN_DATASOURCE_PASSWORD=datahub
9173
- JAVA_OPTS=-Xms1g -Xmx1g
9274
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
75+
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
76+
- UI_INGESTION_ENABLED=true
77+
- ENTITY_SERVICE_ENABLE_RETENTION=true
9378
- MAE_CONSUMER_ENABLED=true
94-
- MCE_CONSUMER_ENABLED=true
79+
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
80+
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
81+
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
82+
- GRAPH_SERVICE_IMPL=elasticsearch
9583
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
84+
- ELASTICSEARCH_PORT=9200
85+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
86+
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
87+
- EBEAN_DATASOURCE_HOST=mysql:3306
9688
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
9789
- PE_CONSUMER_ENABLED=true
9890
hostname: datahub-gms
@@ -131,16 +123,6 @@ services:
131123
- ELASTICSEARCH_PROTOCOL=http
132124
hostname: elasticsearch-setup
133125
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head}
134-
kafka-setup:
135-
container_name: kafka-setup
136-
depends_on:
137-
- broker
138-
- schema-registry
139-
environment:
140-
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
141-
- KAFKA_BOOTSTRAP_SERVER=broker:29092
142-
hostname: kafka-setup
143-
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
144126
mysql:
145127
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
146128
container_name: mysql
@@ -151,7 +133,6 @@ services:
151133
- MYSQL_ROOT_PASSWORD=datahub
152134
hostname: mysql
153135
image: mariadb:10.5.8
154-
# image: mysql:8
155136
ports:
156137
- ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
157138
volumes:
@@ -172,13 +153,13 @@ services:
172153
schema-registry:
173154
container_name: schema-registry
174155
depends_on:
175-
- zookeeper
176156
- broker
177157
environment:
178158
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry
179-
- SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=PLAINTEXT://broker:29092
159+
- SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT
160+
- SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092
180161
hostname: schema-registry
181-
image: confluentinc/cp-schema-registry:7.2.0
162+
image: confluentinc/cp-schema-registry:7.2.2
182163
ports:
183164
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
184165
zookeeper:

‎docker/quickstart/docker-compose-without-neo4j.quickstart.yml

+98-121
Original file line numberDiff line numberDiff line change
@@ -5,195 +5,172 @@ services:
55
broker:
66
container_name: broker
77
depends_on:
8-
- zookeeper
8+
- zookeeper
99
environment:
10-
- KAFKA_BROKER_ID=1
11-
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
12-
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
13-
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
14-
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
15-
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
16-
- KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
17-
- KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
10+
- KAFKA_BROKER_ID=1
11+
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
12+
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
13+
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
14+
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
15+
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
16+
- KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
17+
- KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
1818
hostname: broker
19-
image: confluentinc/cp-kafka:5.4.0
19+
image: confluentinc/cp-kafka:7.2.2
2020
ports:
21-
- ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
21+
- ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
2222
datahub-actions:
2323
depends_on:
24-
- datahub-gms
24+
- datahub-gms
2525
environment:
26-
- DATAHUB_GMS_PROTOCOL=http
27-
- DATAHUB_GMS_HOST=datahub-gms
28-
- DATAHUB_GMS_PORT=8080
29-
- KAFKA_BOOTSTRAP_SERVER=broker:29092
30-
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
31-
- METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
32-
- METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
33-
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
34-
- DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
35-
- KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
36-
- DATAHUB_ACTIONS_SLACK_ENABLED
37-
- DATAHUB_ACTIONS_SLACK_DATAHUB_BASE_URL
38-
- DATAHUB_ACTIONS_SLACK_BOT_TOKEN
39-
- DATAHUB_ACTIONS_SLACK_SIGNING_SECRET
40-
- DATAHUB_ACTIONS_SLACK_CHANNEL
41-
- DATAHUB_ACTIONS_SLACK_SUPPRESS_SYSTEM_ACTIVITY
42-
- DATAHUB_ACTIONS_TEAMS_ENABLED
43-
- DATAHUB_ACTIONS_TEAMS_DATAHUB_BASE_URL
44-
- DATAHUB_ACTIONS_TEAMS_WEBHOOK_URL
45-
- DATAHUB_ACTIONS_TEAMS_SUPPRESS_SYSTEM_ACTIVITY
26+
- DATAHUB_GMS_PROTOCOL=http
27+
- DATAHUB_GMS_HOST=datahub-gms
28+
- DATAHUB_GMS_PORT=8080
29+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
30+
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
31+
- METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
32+
- METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
33+
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
34+
- DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
35+
- KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
4636
hostname: actions
4737
image: acryldata/datahub-actions:${ACTIONS_VERSION:-head}
4838
restart: on-failure:5
4939
datahub-frontend-react:
5040
container_name: datahub-frontend-react
5141
depends_on:
52-
- datahub-gms
42+
- datahub-gms
5343
environment:
54-
- DATAHUB_GMS_HOST=datahub-gms
55-
- DATAHUB_GMS_PORT=8080
56-
- DATAHUB_SECRET=YouKnowNothing
57-
- DATAHUB_APP_VERSION=1.0
58-
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
59-
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf
60-
-Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml
61-
-Dlogback.debug=false -Dpidfile.path=/dev/null
62-
- KAFKA_BOOTSTRAP_SERVER=broker:29092
63-
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
64-
- ELASTIC_CLIENT_HOST=elasticsearch
65-
- ELASTIC_CLIENT_PORT=9200
44+
- DATAHUB_GMS_HOST=datahub-gms
45+
- DATAHUB_GMS_PORT=8080
46+
- DATAHUB_SECRET=YouKnowNothing
47+
- DATAHUB_APP_VERSION=1.0
48+
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
49+
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null
50+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
51+
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
52+
- ELASTIC_CLIENT_HOST=elasticsearch
53+
- ELASTIC_CLIENT_PORT=9200
6654
hostname: datahub-frontend-react
6755
image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head}
6856
ports:
69-
- ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
57+
- ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
7058
volumes:
71-
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
59+
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
7260
datahub-gms:
7361
container_name: datahub-gms
7462
depends_on:
75-
- mysql
63+
- mysql
7664
environment:
77-
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
78-
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
79-
- EBEAN_DATASOURCE_USERNAME=datahub
80-
- EBEAN_DATASOURCE_PASSWORD=datahub
81-
- EBEAN_DATASOURCE_HOST=mysql:3306
82-
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
83-
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
84-
- KAFKA_BOOTSTRAP_SERVER=broker:29092
85-
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
86-
- ELASTICSEARCH_HOST=elasticsearch
87-
- ELASTICSEARCH_PORT=9200
88-
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
89-
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
90-
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
91-
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
92-
- GRAPH_SERVICE_IMPL=elasticsearch
93-
- JAVA_OPTS=-Xms1g -Xmx1g
94-
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
95-
- MAE_CONSUMER_ENABLED=true
96-
- MCE_CONSUMER_ENABLED=true
97-
- PE_CONSUMER_ENABLED=true
98-
- UI_INGESTION_ENABLED=true
99-
- ENTITY_SERVICE_ENABLE_RETENTION=true
65+
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
66+
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
67+
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
68+
- UI_INGESTION_ENABLED=true
69+
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
70+
- GRAPH_SERVICE_IMPL=elasticsearch
71+
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
72+
- MCE_CONSUMER_ENABLED=true
73+
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
74+
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
75+
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
76+
- MAE_CONSUMER_ENABLED=true
77+
- JAVA_OPTS=-Xms1g -Xmx1g
78+
- EBEAN_DATASOURCE_HOST=mysql:3306
79+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
80+
- ENTITY_SERVICE_ENABLE_RETENTION=true
81+
- EBEAN_DATASOURCE_USERNAME=datahub
82+
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
83+
- ELASTICSEARCH_PORT=9200
84+
- ELASTICSEARCH_HOST=elasticsearch
85+
- PE_CONSUMER_ENABLED=true
86+
- EBEAN_DATASOURCE_PASSWORD=datahub
87+
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
10088
hostname: datahub-gms
10189
image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head}
10290
ports:
103-
- ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
91+
- ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
10492
volumes:
105-
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
93+
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
10694
elasticsearch:
10795
container_name: elasticsearch
10896
environment:
109-
- discovery.type=single-node
110-
- xpack.security.enabled=false
111-
- ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true
97+
- discovery.type=single-node
98+
- xpack.security.enabled=false
99+
- ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true
112100
healthcheck:
113101
retries: 4
114102
start_period: 2m
115103
test:
116-
- CMD-SHELL
117-
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s'
118-
|| exit 1
104+
- CMD-SHELL
105+
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1
119106
hostname: elasticsearch
120107
image: elasticsearch:7.9.3
121108
mem_limit: 1g
122109
ports:
123-
- ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200
110+
- ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200
124111
volumes:
125-
- esdata:/usr/share/elasticsearch/data
112+
- esdata:/usr/share/elasticsearch/data
126113
elasticsearch-setup:
127114
container_name: elasticsearch-setup
128115
depends_on:
129-
- elasticsearch
116+
- elasticsearch
130117
environment:
131-
- ELASTICSEARCH_HOST=elasticsearch
132-
- ELASTICSEARCH_PORT=9200
133-
- ELASTICSEARCH_PROTOCOL=http
118+
- ELASTICSEARCH_HOST=elasticsearch
119+
- ELASTICSEARCH_PORT=9200
120+
- ELASTICSEARCH_PROTOCOL=http
134121
hostname: elasticsearch-setup
135122
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head}
136-
kafka-setup:
137-
container_name: kafka-setup
138-
depends_on:
139-
- broker
140-
- schema-registry
141-
environment:
142-
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
143-
- KAFKA_BOOTSTRAP_SERVER=broker:29092
144-
hostname: kafka-setup
145-
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
146123
mysql:
147-
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
124+
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
148125
container_name: mysql
149126
environment:
150-
- MYSQL_DATABASE=datahub
151-
- MYSQL_USER=datahub
152-
- MYSQL_PASSWORD=datahub
153-
- MYSQL_ROOT_PASSWORD=datahub
127+
- MYSQL_DATABASE=datahub
128+
- MYSQL_USER=datahub
129+
- MYSQL_PASSWORD=datahub
130+
- MYSQL_ROOT_PASSWORD=datahub
154131
hostname: mysql
155132
image: mysql:5.7
156133
ports:
157-
- ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
134+
- ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
158135
volumes:
159-
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
160-
- mysqldata:/var/lib/mysql
136+
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
137+
- mysqldata:/var/lib/mysql
161138
mysql-setup:
162139
container_name: mysql-setup
163140
depends_on:
164-
- mysql
141+
- mysql
165142
environment:
166-
- MYSQL_HOST=mysql
167-
- MYSQL_PORT=3306
168-
- MYSQL_USERNAME=datahub
169-
- MYSQL_PASSWORD=datahub
170-
- DATAHUB_DB_NAME=datahub
143+
- MYSQL_HOST=mysql
144+
- MYSQL_PORT=3306
145+
- MYSQL_USERNAME=datahub
146+
- MYSQL_PASSWORD=datahub
147+
- DATAHUB_DB_NAME=datahub
171148
hostname: mysql-setup
172149
image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head}
173150
schema-registry:
174151
container_name: schema-registry
175152
depends_on:
176-
- zookeeper
177-
- broker
153+
- broker
178154
environment:
179-
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry
180-
- SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181
155+
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry
156+
- SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT
157+
- SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092
181158
hostname: schema-registry
182-
image: confluentinc/cp-schema-registry:5.4.0
159+
image: confluentinc/cp-schema-registry:7.2.2
183160
ports:
184-
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
161+
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
185162
zookeeper:
186163
container_name: zookeeper
187164
environment:
188-
- ZOOKEEPER_CLIENT_PORT=2181
189-
- ZOOKEEPER_TICK_TIME=2000
165+
- ZOOKEEPER_CLIENT_PORT=2181
166+
- ZOOKEEPER_TICK_TIME=2000
190167
hostname: zookeeper
191-
image: confluentinc/cp-zookeeper:5.4.0
168+
image: confluentinc/cp-zookeeper:7.2.2
192169
ports:
193-
- ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
170+
- ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
194171
volumes:
195-
- zkdata:/var/lib/zookeeper
196-
version: '2.3'
172+
- zkdata:/var/lib/zookeeper
173+
version: "2.3"
197174
volumes:
198175
esdata: null
199176
mysqldata: null

‎docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -24,16 +24,16 @@ services:
2424
datahub-mce-consumer:
2525
container_name: datahub-mce-consumer
2626
environment:
27-
- MCE_CONSUMER_ENABLED=true
28-
- KAFKA_BOOTSTRAP_SERVER=broker:29092
29-
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
3027
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
3128
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
29+
- MCE_CONSUMER_ENABLED=true
3230
- EBEAN_DATASOURCE_USERNAME=datahub
3331
- EBEAN_DATASOURCE_PASSWORD=datahub
3432
- EBEAN_DATASOURCE_HOST=mysql:3306
35-
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
33+
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
3634
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
35+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
36+
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
3737
- ELASTICSEARCH_HOST=elasticsearch
3838
- ELASTICSEARCH_PORT=9200
3939
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL

‎docker/quickstart/docker-compose.consumers.quickstart.yml

+11-9
Original file line numberDiff line numberDiff line change
@@ -29,27 +29,29 @@ services:
2929
- 9091:9091
3030
datahub-mce-consumer:
3131
container_name: datahub-mce-consumer
32+
depends_on:
33+
- neo4j
3234
environment:
33-
- MCE_CONSUMER_ENABLED=true
34-
- KAFKA_BOOTSTRAP_SERVER=broker:29092
35-
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
3635
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
3736
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
37+
- NEO4J_HOST=http://neo4j:7474
38+
- NEO4J_URI=bolt://neo4j
39+
- NEO4J_USERNAME=neo4j
40+
- NEO4J_PASSWORD=datahub
41+
- GRAPH_SERVICE_IMPL=neo4j
42+
- MCE_CONSUMER_ENABLED=true
3843
- EBEAN_DATASOURCE_USERNAME=datahub
3944
- EBEAN_DATASOURCE_PASSWORD=datahub
4045
- EBEAN_DATASOURCE_HOST=mysql:3306
4146
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
4247
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
48+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
49+
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
4350
- ELASTICSEARCH_HOST=elasticsearch
4451
- ELASTICSEARCH_PORT=9200
4552
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
46-
- NEO4J_HOST=http://neo4j:7474
47-
- NEO4J_URI=bolt://neo4j
48-
- NEO4J_USERNAME=neo4j
49-
- NEO4J_PASSWORD=datahub
50-
- JAVA_OPTS=-Xms1g -Xmx1g
5153
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
52-
- GRAPH_SERVICE_IMPL=neo4j
54+
- JAVA_OPTS=-Xms1g -Xmx1g
5355
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml
5456
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
5557
- DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing

‎docker/quickstart/docker-compose.kafka-setup.quickstart.yml

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
services:
2+
kafka-setup:
3+
container_name: kafka-setup
4+
depends_on:
5+
- broker
6+
- schema-registry
7+
environment:
8+
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
9+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
10+
hostname: kafka-setup
11+
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
12+
version: '2.3'

‎docker/quickstart/docker-compose.monitoring.quickstart.yml

+25-25
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,47 @@
11
services:
22
datahub-frontend-react:
33
environment:
4-
- ENABLE_PROMETHEUS=true
5-
- ENABLE_OTEL=true
6-
- OTEL_TRACES_EXPORTER=jaeger
7-
- OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250
8-
- OTEL_METRICS_EXPORTER=none
9-
- OTEL_SERVICE_NAME=datahub-gms
4+
- ENABLE_PROMETHEUS=true
5+
- ENABLE_OTEL=true
6+
- OTEL_TRACES_EXPORTER=jaeger
7+
- OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250
8+
- OTEL_METRICS_EXPORTER=none
9+
- OTEL_SERVICE_NAME=datahub-gms
1010
ports:
11-
- '4318'
11+
- "4318"
1212
datahub-gms:
1313
environment:
14-
- ENABLE_PROMETHEUS=true
15-
- ENABLE_OTEL=true
16-
- OTEL_TRACES_EXPORTER=jaeger
17-
- OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250
18-
- OTEL_METRICS_EXPORTER=none
19-
- OTEL_SERVICE_NAME=datahub-gms
14+
- ENABLE_PROMETHEUS=true
15+
- ENABLE_OTEL=true
16+
- OTEL_TRACES_EXPORTER=jaeger
17+
- OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250
18+
- OTEL_METRICS_EXPORTER=none
19+
- OTEL_SERVICE_NAME=datahub-gms
2020
ports:
21-
- '4318'
21+
- "4318"
2222
grafana:
2323
depends_on:
24-
- prometheus
24+
- prometheus
2525
image: grafana/grafana:9.1.4
2626
ports:
27-
- 3001:3000
27+
- 3001:3000
2828
volumes:
29-
- grafana-storage:/var/lib/grafana
30-
- ../monitoring/grafana/datasources:/etc/grafana/provisioning/datasources
31-
- ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards
29+
- grafana-storage:/var/lib/grafana
30+
- ../monitoring/grafana/datasources:/etc/grafana/provisioning/datasources
31+
- ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards
3232
jaeger-all-in-one:
3333
image: jaegertracing/all-in-one:latest
3434
ports:
35-
- 16686:16686
36-
- '14268'
37-
- '14250'
35+
- 16686:16686
36+
- "14268"
37+
- "14250"
3838
prometheus:
3939
container_name: prometheus
4040
image: prom/prometheus:latest
4141
ports:
42-
- 9089:9090
42+
- 9089:9090
4343
volumes:
44-
- ../monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml
45-
version: '2.3'
44+
- ../monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml
45+
version: "2.3"
4646
volumes:
4747
grafana-storage: null

‎docker/quickstart/docker-compose.quickstart.yml

+112-134
Original file line numberDiff line numberDiff line change
@@ -5,216 +5,194 @@ services:
55
broker:
66
container_name: broker
77
depends_on:
8-
- zookeeper
8+
- zookeeper
99
environment:
10-
- KAFKA_BROKER_ID=1
11-
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
12-
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
13-
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
14-
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
15-
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
16-
- KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
17-
- KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
10+
- KAFKA_BROKER_ID=1
11+
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
12+
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
13+
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
14+
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
15+
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
16+
- KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
17+
- KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
1818
hostname: broker
19-
image: confluentinc/cp-kafka:5.4.0
19+
image: confluentinc/cp-kafka:7.2.2
2020
ports:
21-
- ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
21+
- ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
2222
volumes:
23-
- broker:/var/lib/kafka/data/
23+
- broker:/var/lib/kafka/data/
2424
datahub-actions:
2525
depends_on:
26-
- datahub-gms
26+
- datahub-gms
2727
environment:
28-
- DATAHUB_GMS_PROTOCOL=http
29-
- DATAHUB_GMS_HOST=datahub-gms
30-
- DATAHUB_GMS_PORT=8080
31-
- KAFKA_BOOTSTRAP_SERVER=broker:29092
32-
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
33-
- METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
34-
- METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
35-
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
36-
- DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
37-
- KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
38-
- DATAHUB_ACTIONS_SLACK_ENABLED
39-
- DATAHUB_ACTIONS_SLACK_DATAHUB_BASE_URL
40-
- DATAHUB_ACTIONS_SLACK_BOT_TOKEN
41-
- DATAHUB_ACTIONS_SLACK_SIGNING_SECRET
42-
- DATAHUB_ACTIONS_SLACK_CHANNEL
43-
- DATAHUB_ACTIONS_SLACK_SUPPRESS_SYSTEM_ACTIVITY
44-
- DATAHUB_ACTIONS_TEAMS_ENABLED
45-
- DATAHUB_ACTIONS_TEAMS_DATAHUB_BASE_URL
46-
- DATAHUB_ACTIONS_TEAMS_WEBHOOK_URL
47-
- DATAHUB_ACTIONS_TEAMS_SUPPRESS_SYSTEM_ACTIVITY
28+
- DATAHUB_GMS_PROTOCOL=http
29+
- DATAHUB_GMS_HOST=datahub-gms
30+
- DATAHUB_GMS_PORT=8080
31+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
32+
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
33+
- METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
34+
- METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
35+
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
36+
- DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
37+
- KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
4838
hostname: actions
4939
image: acryldata/datahub-actions:${ACTIONS_VERSION:-head}
5040
restart: on-failure:5
5141
datahub-frontend-react:
5242
container_name: datahub-frontend-react
5343
depends_on:
54-
- datahub-gms
44+
- datahub-gms
5545
environment:
56-
- DATAHUB_GMS_HOST=datahub-gms
57-
- DATAHUB_GMS_PORT=8080
58-
- DATAHUB_SECRET=YouKnowNothing
59-
- DATAHUB_APP_VERSION=1.0
60-
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
61-
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf
62-
-Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml
63-
-Dlogback.debug=false -Dpidfile.path=/dev/null
64-
- KAFKA_BOOTSTRAP_SERVER=broker:29092
65-
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
66-
- ELASTIC_CLIENT_HOST=elasticsearch
67-
- ELASTIC_CLIENT_PORT=9200
46+
- DATAHUB_GMS_HOST=datahub-gms
47+
- DATAHUB_GMS_PORT=8080
48+
- DATAHUB_SECRET=YouKnowNothing
49+
- DATAHUB_APP_VERSION=1.0
50+
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
51+
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null
52+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
53+
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
54+
- ELASTIC_CLIENT_HOST=elasticsearch
55+
- ELASTIC_CLIENT_PORT=9200
6856
hostname: datahub-frontend-react
6957
image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head}
7058
ports:
71-
- ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
59+
- ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
7260
volumes:
73-
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
61+
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
7462
datahub-gms:
7563
container_name: datahub-gms
7664
depends_on:
77-
- mysql
65+
- mysql
66+
- neo4j
7867
environment:
79-
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
80-
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
81-
- EBEAN_DATASOURCE_USERNAME=datahub
82-
- EBEAN_DATASOURCE_PASSWORD=datahub
83-
- EBEAN_DATASOURCE_HOST=mysql:3306
84-
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
85-
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
86-
- KAFKA_BOOTSTRAP_SERVER=broker:29092
87-
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
88-
- ELASTICSEARCH_HOST=elasticsearch
89-
- ELASTICSEARCH_PORT=9200
90-
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
91-
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
92-
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
93-
- NEO4J_HOST=http://neo4j:7474
94-
- NEO4J_URI=bolt://neo4j
95-
- NEO4J_USERNAME=neo4j
96-
- NEO4J_PASSWORD=datahub
97-
- JAVA_OPTS=-Xms1g -Xmx1g
98-
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
99-
- GRAPH_SERVICE_IMPL=neo4j
100-
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
101-
- ENTITY_SERVICE_ENABLE_RETENTION=true
102-
- MAE_CONSUMER_ENABLED=true
103-
- MCE_CONSUMER_ENABLED=true
104-
- PE_CONSUMER_ENABLED=true
105-
- UI_INGESTION_ENABLED=true
106-
- METADATA_SERVICE_AUTH_ENABLED=false
68+
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
69+
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
70+
- EBEAN_DATASOURCE_USERNAME=datahub
71+
- EBEAN_DATASOURCE_PASSWORD=datahub
72+
- EBEAN_DATASOURCE_HOST=mysql:3306
73+
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
74+
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
75+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
76+
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
77+
- ELASTICSEARCH_HOST=elasticsearch
78+
- ELASTICSEARCH_PORT=9200
79+
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
80+
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
81+
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
82+
- NEO4J_HOST=http://neo4j:7474
83+
- NEO4J_URI=bolt://neo4j
84+
- NEO4J_USERNAME=neo4j
85+
- NEO4J_PASSWORD=datahub
86+
- JAVA_OPTS=-Xms1g -Xmx1g
87+
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
88+
- GRAPH_SERVICE_IMPL=neo4j
89+
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
90+
- ENTITY_SERVICE_ENABLE_RETENTION=true
91+
- MAE_CONSUMER_ENABLED=true
92+
- MCE_CONSUMER_ENABLED=true
93+
- PE_CONSUMER_ENABLED=true
94+
- UI_INGESTION_ENABLED=true
95+
- METADATA_SERVICE_AUTH_ENABLED=false
10796
hostname: datahub-gms
10897
image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head}
10998
ports:
110-
- ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
99+
- ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
111100
volumes:
112-
- ${HOME}/.datahub/plugins/:/etc/datahub/plugins
113-
- ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources
101+
- ${HOME}/.datahub/plugins/:/etc/datahub/plugins
102+
- ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources
114103
elasticsearch:
115104
container_name: elasticsearch
116105
environment:
117-
- discovery.type=single-node
118-
- xpack.security.enabled=false
119-
- ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true
106+
- discovery.type=single-node
107+
- xpack.security.enabled=false
108+
- ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true
120109
healthcheck:
121110
retries: 4
122111
start_period: 2m
123112
test:
124-
- CMD-SHELL
125-
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s'
126-
|| exit 1
113+
- CMD-SHELL
114+
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1
127115
hostname: elasticsearch
128116
image: elasticsearch:7.9.3
129117
mem_limit: 1g
130118
ports:
131-
- ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200
119+
- ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200
132120
volumes:
133-
- esdata:/usr/share/elasticsearch/data
121+
- esdata:/usr/share/elasticsearch/data
134122
elasticsearch-setup:
135123
container_name: elasticsearch-setup
136124
depends_on:
137-
- elasticsearch
125+
- elasticsearch
138126
environment:
139-
- ELASTICSEARCH_HOST=elasticsearch
140-
- ELASTICSEARCH_PORT=9200
141-
- ELASTICSEARCH_PROTOCOL=http
127+
- ELASTICSEARCH_HOST=elasticsearch
128+
- ELASTICSEARCH_PORT=9200
129+
- ELASTICSEARCH_PROTOCOL=http
142130
hostname: elasticsearch-setup
143131
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head}
144-
kafka-setup:
145-
container_name: kafka-setup
146-
depends_on:
147-
- broker
148-
- schema-registry
149-
environment:
150-
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
151-
- KAFKA_BOOTSTRAP_SERVER=broker:29092
152-
hostname: kafka-setup
153-
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
154132
mysql:
155-
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
133+
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
156134
container_name: mysql
157135
environment:
158-
- MYSQL_DATABASE=datahub
159-
- MYSQL_USER=datahub
160-
- MYSQL_PASSWORD=datahub
161-
- MYSQL_ROOT_PASSWORD=datahub
136+
- MYSQL_DATABASE=datahub
137+
- MYSQL_USER=datahub
138+
- MYSQL_PASSWORD=datahub
139+
- MYSQL_ROOT_PASSWORD=datahub
162140
hostname: mysql
163141
image: mysql:5.7
164142
ports:
165-
- ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
143+
- ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
166144
volumes:
167-
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
168-
- mysqldata:/var/lib/mysql
145+
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
146+
- mysqldata:/var/lib/mysql
169147
mysql-setup:
170148
container_name: mysql-setup
171149
depends_on:
172-
- mysql
150+
- mysql
173151
environment:
174-
- MYSQL_HOST=mysql
175-
- MYSQL_PORT=3306
176-
- MYSQL_USERNAME=datahub
177-
- MYSQL_PASSWORD=datahub
178-
- DATAHUB_DB_NAME=datahub
152+
- MYSQL_HOST=mysql
153+
- MYSQL_PORT=3306
154+
- MYSQL_USERNAME=datahub
155+
- MYSQL_PASSWORD=datahub
156+
- DATAHUB_DB_NAME=datahub
179157
hostname: mysql-setup
180158
image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head}
181159
neo4j:
182160
container_name: neo4j
183161
environment:
184-
- NEO4J_AUTH=neo4j/datahub
185-
- NEO4J_dbms_default__database=graph.db
186-
- NEO4J_dbms_allow__upgrade=true
162+
- NEO4J_AUTH=neo4j/datahub
163+
- NEO4J_dbms_default__database=graph.db
164+
- NEO4J_dbms_allow__upgrade=true
187165
hostname: neo4j
188166
image: neo4j:4.4.9-community
189167
ports:
190-
- ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474
191-
- ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687
168+
- ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474
169+
- ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687
192170
volumes:
193-
- neo4jdata:/data
171+
- neo4jdata:/data
194172
schema-registry:
195173
container_name: schema-registry
196174
depends_on:
197-
- zookeeper
198-
- broker
175+
- broker
199176
environment:
200-
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry
201-
- SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181
177+
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry
178+
- SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT
179+
- SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092
202180
hostname: schema-registry
203-
image: confluentinc/cp-schema-registry:5.4.0
181+
image: confluentinc/cp-schema-registry:7.2.2
204182
ports:
205-
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
183+
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
206184
zookeeper:
207185
container_name: zookeeper
208186
environment:
209-
- ZOOKEEPER_CLIENT_PORT=2181
210-
- ZOOKEEPER_TICK_TIME=2000
187+
- ZOOKEEPER_CLIENT_PORT=2181
188+
- ZOOKEEPER_TICK_TIME=2000
211189
hostname: zookeeper
212-
image: confluentinc/cp-zookeeper:5.4.0
190+
image: confluentinc/cp-zookeeper:7.2.2
213191
ports:
214-
- ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
192+
- ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
215193
volumes:
216-
- zkdata:/var/lib/zookeeper
217-
version: '2.3'
194+
- zkdata:/var/lib/zookeeper
195+
version: "2.3"
218196
volumes:
219197
broker: null
220198
esdata: null

‎docker/quickstart/generate_and_compare.sh

+4-1
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,16 @@ source venv/bin/activate
1414
pip install -r requirements.txt
1515
python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml temp.quickstart.yml
1616
python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml temp-without-neo4j.quickstart.yml
17+
python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml ../docker-compose.m1.yml temp-m1.quickstart.yml
18+
python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml ../docker-compose-without-neo4j.m1.yml temp-without-neo4j-m1.quickstart.yml
1719
python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml temp.monitoring.quickstart.yml
1820
python generate_docker_quickstart.py ../docker-compose.consumers.yml temp.consumers.quickstart.yml
1921
python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml temp.consumers-without-neo4j.quickstart.yml
2022

2123
for flavour in "${FLAVOURS[@]}"
2224
do
23-
if cmp docker-compose$flavour.quickstart.yml temp$flavour.quickstart.yml; then
25+
26+
if cmp <(yq -i -P 'sort_keys(..)' docker-compose$flavour.quickstart.yml) <(yq -i -P 'sort_keys(..)' temp$flavour.quickstart.yml); then
2427
echo "docker-compose$flavour.quickstart.yml is up to date."
2528
else
2629
echo "docker-compose$flavour.quickstart.yml is out of date."

‎docker/quickstart/generate_docker_quickstart.py

+29-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import os
2-
from collections import OrderedDict
32
from collections.abc import Mapping
43

54
import click
@@ -27,10 +26,14 @@ def dict_merge(dct, merge_dct):
2726
for k, v in merge_dct.items():
2827
if k in dct and isinstance(dct[k], dict) and isinstance(merge_dct[k], Mapping):
2928
dict_merge(dct[k], merge_dct[k])
29+
elif k in dct and isinstance(dct[k], list):
30+
a = set(dct[k])
31+
b = set(merge_dct[k])
32+
if a != b:
33+
dct[k] = list(a.union(b))
3034
else:
3135
dct[k] = merge_dct[k]
3236

33-
3437
def modify_docker_config(base_path, docker_yaml_config):
3538
# 0. Filter out services to be omitted.
3639
for key in list(docker_yaml_config["services"]):
@@ -80,7 +83,7 @@ def modify_docker_config(base_path, docker_yaml_config):
8083
elif volumes[i].startswith("./"):
8184
volumes[i] = "." + volumes[i]
8285

83-
# 9. Set docker compose version to 2.
86+
# 10. Set docker compose version to 2.
8487
# We need at least this version, since we use features like start_period for
8588
# healthchecks and shell-like variable interpolation.
8689
docker_yaml_config["version"] = "2.3"
@@ -113,6 +116,9 @@ def generate(compose_files, output_file) -> None:
113116
for modified_file in modified_files:
114117
dict_merge(merged_docker_config, modified_file)
115118

119+
# Dedup env vars, last wins
120+
dedup_env_vars(merged_docker_config)
121+
116122
# Write output file
117123
output_dir = os.path.dirname(output_file)
118124
if len(output_dir) and not os.path.exists(output_dir):
@@ -127,5 +133,25 @@ def generate(compose_files, output_file) -> None:
127133
print(f"Successfully generated {output_file}.")
128134

129135

136+
def dedup_env_vars(merged_docker_config):
137+
for service in merged_docker_config['services']:
138+
if 'environment' in merged_docker_config['services'][service]:
139+
lst = merged_docker_config['services'][service]['environment']
140+
if lst is not None:
141+
# use a set to cache duplicates
142+
caches = set()
143+
results = []
144+
for item in lst:
145+
partitions = item.rpartition('=')
146+
prefix = partitions[0]
147+
suffix = partitions[1]
148+
# check whether prefix already exists
149+
if prefix not in caches and suffix != "":
150+
results.append(item)
151+
caches.add(prefix)
152+
if set(lst) != set(results):
153+
merged_docker_config['services'][service]['environment'] = results
154+
155+
130156
if __name__ == "__main__":
131157
generate()

‎docker/quickstart/generate_docker_quickstart.sh

+3
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ source venv/bin/activate
1111
pip install -r requirements.txt
1212
python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml docker-compose.quickstart.yml
1313
python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml docker-compose-without-neo4j.quickstart.yml
14+
python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml ../docker-compose.m1.yml docker-compose-m1.quickstart.yml
15+
python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml ../docker-compose-without-neo4j.m1.yml docker-compose-without-neo4j-m1.quickstart.yml
1416
python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml docker-compose.monitoring.quickstart.yml
1517
python generate_docker_quickstart.py ../docker-compose.consumers.yml docker-compose.consumers.quickstart.yml
1618
python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml docker-compose.consumers-without-neo4j.quickstart.yml
19+
python generate_docker_quickstart.py ../docker-compose.kafka-setup.yml docker-compose.kafka-setup.quickstart.yml

‎docker/schema-registry/env/docker.env

+6-4
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
SCHEMA_REGISTRY_HOST_NAME=schemaregistry
2-
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181
2+
SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT
3+
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092
34

4-
# Uncomment to customize the Schema Registry kafka store connection
5-
# SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT
6-
# SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092
5+
# Uncomment to customize the Schema Registry kafka store connection
76
# ZOOKEEPER_SASL_ENABLED=false
87
# KAFKA_OPTS=-Xms1g -Xmx1g
98
# SCHEMA_REGISTRY_JMX_OPTS=-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false
9+
10+
# Uncomment to use schema registry < v5.4.0
11+
# SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181

‎metadata-ingestion/src/datahub/cli/docker_check.py

-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
"elasticsearch",
1010
"datahub-gms",
1111
"datahub-frontend-react",
12-
"kafka-setup",
1312
"schema-registry",
1413
"broker",
1514
"zookeeper",

‎metadata-ingestion/src/datahub/cli/docker_cli.py

+119-62
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,10 @@
4343
ELASTIC_QUICKSTART_COMPOSE_FILE = (
4444
"docker/quickstart/docker-compose-without-neo4j.quickstart.yml"
4545
)
46-
M1_QUICKSTART_COMPOSE_FILE = (
46+
NEO4J_AND_ELASTIC_M1_QUICKSTART_COMPOSE_FILE = (
47+
"docker/quickstart/docker-compose-m1.quickstart.yml"
48+
)
49+
ELASTIC_M1_QUICKSTART_COMPOSE_FILE = (
4750
"docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml"
4851
)
4952
CONSUMERS_QUICKSTART_COMPOSE_FILE = (
@@ -52,14 +55,21 @@
5255
ELASTIC_CONSUMERS_QUICKSTART_COMPOSE_FILE = (
5356
"docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml"
5457
)
55-
58+
KAFKA_SETUP_QUICKSTART_COMPOSE_FILE = (
59+
"docker/quickstart/docker-compose.kafka-setup.quickstart.yml"
60+
)
5661
NEO4J_AND_ELASTIC_QUICKSTART_COMPOSE_URL = (
5762
f"{DOCKER_COMPOSE_BASE}/{NEO4J_AND_ELASTIC_QUICKSTART_COMPOSE_FILE}"
5863
)
5964
ELASTIC_QUICKSTART_COMPOSE_URL = (
6065
f"{DOCKER_COMPOSE_BASE}/{ELASTIC_QUICKSTART_COMPOSE_FILE}"
6166
)
62-
M1_QUICKSTART_COMPOSE_URL = f"{DOCKER_COMPOSE_BASE}/{M1_QUICKSTART_COMPOSE_FILE}"
67+
NEO4J_AND_ELASTIC_M1_QUICKSTART_COMPOSE_URL = (
68+
f"{DOCKER_COMPOSE_BASE}/{NEO4J_AND_ELASTIC_M1_QUICKSTART_COMPOSE_FILE}"
69+
)
70+
ELASTIC_M1_QUICKSTART_COMPOSE_URL = (
71+
f"{DOCKER_COMPOSE_BASE}/{ELASTIC_M1_QUICKSTART_COMPOSE_FILE}"
72+
)
6373

6474

6575
class Architectures(Enum):
@@ -166,7 +176,7 @@ def should_use_neo4j_for_graph_service(graph_service_override: Optional[str]) ->
166176
click.echo(
167177
"No Datahub Neo4j volume found, starting with elasticsearch as graph service.\n"
168178
"To use neo4j as a graph backend, run \n"
169-
"`datahub docker quickstart --quickstart-compose-file ./docker/quickstart/docker-compose.quickstart.yml`"
179+
"`datahub docker quickstart --graph-service-impl neo4j`"
170180
"\nfrom the root of the datahub repo\n"
171181
)
172182
return False
@@ -581,6 +591,13 @@ def detect_quickstart_arch(arch: Optional[str]) -> Architectures:
581591
default=False,
582592
help="Launches MAE & MCE consumers as stand alone docker containers",
583593
)
594+
@click.option(
595+
"--kafka-setup",
596+
required=False,
597+
is_flag=True,
598+
default=False,
599+
help="Launches Kafka setup job as part of the compose deployment",
600+
)
584601
@click.option(
585602
"--arch",
586603
required=False,
@@ -608,6 +625,7 @@ def quickstart(
608625
restore_indices: bool,
609626
no_restore_indices: bool,
610627
standalone_consumers: bool,
628+
kafka_setup: bool,
611629
arch: Optional[str],
612630
) -> None:
613631
"""Start an instance of DataHub locally using docker-compose.
@@ -650,70 +668,21 @@ def quickstart(
650668
auth_resources_folder = Path(DATAHUB_ROOT_FOLDER) / "plugins/auth/resources"
651669
os.makedirs(auth_resources_folder, exist_ok=True)
652670

653-
default_quickstart_compose_file = _get_default_quickstart_compose_file()
671+
quickstart_compose_file_name = _get_default_quickstart_compose_file()
654672
if stop:
655673
_attempt_stop(quickstart_compose_file)
656674
return
657675
elif not quickstart_compose_file:
658-
# download appropriate quickstart file
659-
should_use_neo4j = should_use_neo4j_for_graph_service(graph_service_impl)
660-
if should_use_neo4j and is_arch_m1(quickstart_arch):
661-
click.secho(
662-
"Running with neo4j on M1 is not currently supported, will be using elasticsearch as graph",
663-
fg="red",
664-
)
665-
github_file = (
666-
NEO4J_AND_ELASTIC_QUICKSTART_COMPOSE_URL
667-
if should_use_neo4j and not is_arch_m1(quickstart_arch)
668-
else ELASTIC_QUICKSTART_COMPOSE_URL
669-
if not is_arch_m1(quickstart_arch)
670-
else M1_QUICKSTART_COMPOSE_URL
676+
print("compose file name", quickstart_compose_file_name)
677+
download_compose_files(
678+
quickstart_compose_file_name,
679+
quickstart_compose_file,
680+
graph_service_impl,
681+
kafka_setup,
682+
quickstart_arch,
683+
standalone_consumers,
671684
)
672685

673-
# also allow local files
674-
request_session = requests.Session()
675-
request_session.mount("file://", FileAdapter())
676-
677-
with open(
678-
default_quickstart_compose_file, "wb"
679-
) if default_quickstart_compose_file else tempfile.NamedTemporaryFile(
680-
suffix=".yml", delete=False
681-
) as tmp_file:
682-
path = pathlib.Path(tmp_file.name)
683-
quickstart_compose_file.append(path)
684-
click.echo(f"Fetching docker-compose file {github_file} from GitHub")
685-
# Download the quickstart docker-compose file from GitHub.
686-
quickstart_download_response = request_session.get(github_file)
687-
quickstart_download_response.raise_for_status()
688-
tmp_file.write(quickstart_download_response.content)
689-
logger.debug(f"Copied to {path}")
690-
691-
if standalone_consumers:
692-
consumer_github_file = (
693-
f"{DOCKER_COMPOSE_BASE}/{CONSUMERS_QUICKSTART_COMPOSE_FILE}"
694-
if should_use_neo4j
695-
else f"{DOCKER_COMPOSE_BASE}/{ELASTIC_CONSUMERS_QUICKSTART_COMPOSE_FILE}"
696-
)
697-
698-
default_consumer_compose_file = (
699-
Path(DATAHUB_ROOT_FOLDER) / "quickstart/docker-compose.consumers.yml"
700-
)
701-
with open(
702-
default_consumer_compose_file, "wb"
703-
) if default_consumer_compose_file else tempfile.NamedTemporaryFile(
704-
suffix=".yml", delete=False
705-
) as tmp_file:
706-
path = pathlib.Path(tmp_file.name)
707-
quickstart_compose_file.append(path)
708-
click.echo(
709-
f"Fetching consumer docker-compose file {consumer_github_file} from GitHub"
710-
)
711-
# Download the quickstart docker-compose file from GitHub.
712-
quickstart_download_response = request_session.get(consumer_github_file)
713-
quickstart_download_response.raise_for_status()
714-
tmp_file.write(quickstart_download_response.content)
715-
logger.debug(f"Copied to {path}")
716-
717686
# set version
718687
_set_environment_variables(
719688
version=version,
@@ -833,6 +802,94 @@ def quickstart(
833802
)
834803

835804

805+
def download_compose_files(
806+
quickstart_compose_file_name,
807+
quickstart_compose_file_list,
808+
graph_service_impl,
809+
kafka_setup,
810+
quickstart_arch,
811+
standalone_consumers,
812+
):
813+
# download appropriate quickstart file
814+
should_use_neo4j = should_use_neo4j_for_graph_service(graph_service_impl)
815+
if should_use_neo4j:
816+
github_file = (
817+
NEO4J_AND_ELASTIC_QUICKSTART_COMPOSE_URL
818+
if not is_arch_m1(quickstart_arch)
819+
else NEO4J_AND_ELASTIC_M1_QUICKSTART_COMPOSE_URL
820+
)
821+
else:
822+
github_file = (
823+
ELASTIC_QUICKSTART_COMPOSE_URL
824+
if not is_arch_m1(quickstart_arch)
825+
else ELASTIC_M1_QUICKSTART_COMPOSE_URL
826+
)
827+
# also allow local files
828+
request_session = requests.Session()
829+
request_session.mount("file://", FileAdapter())
830+
with open(
831+
quickstart_compose_file_name, "wb"
832+
) if quickstart_compose_file_name else tempfile.NamedTemporaryFile(
833+
suffix=".yml", delete=False
834+
) as tmp_file:
835+
path = pathlib.Path(tmp_file.name)
836+
quickstart_compose_file_list.append(path)
837+
click.echo(f"Fetching docker-compose file {github_file} from GitHub")
838+
# Download the quickstart docker-compose file from GitHub.
839+
quickstart_download_response = request_session.get(github_file)
840+
quickstart_download_response.raise_for_status()
841+
tmp_file.write(quickstart_download_response.content)
842+
logger.debug(f"Copied to {path}")
843+
if standalone_consumers:
844+
consumer_github_file = (
845+
f"{DOCKER_COMPOSE_BASE}/{CONSUMERS_QUICKSTART_COMPOSE_FILE}"
846+
if should_use_neo4j
847+
else f"{DOCKER_COMPOSE_BASE}/{ELASTIC_CONSUMERS_QUICKSTART_COMPOSE_FILE}"
848+
)
849+
850+
default_consumer_compose_file = (
851+
Path(DATAHUB_ROOT_FOLDER) / "quickstart/docker-compose.consumers.yml"
852+
)
853+
with open(
854+
default_consumer_compose_file, "wb"
855+
) if default_consumer_compose_file else tempfile.NamedTemporaryFile(
856+
suffix=".yml", delete=False
857+
) as tmp_file:
858+
path = pathlib.Path(tmp_file.name)
859+
quickstart_compose_file_list.append(path)
860+
click.echo(
861+
f"Fetching consumer docker-compose file {consumer_github_file} from GitHub"
862+
)
863+
# Download the quickstart docker-compose file from GitHub.
864+
quickstart_download_response = request_session.get(consumer_github_file)
865+
quickstart_download_response.raise_for_status()
866+
tmp_file.write(quickstart_download_response.content)
867+
logger.debug(f"Copied to {path}")
868+
if kafka_setup:
869+
kafka_setup_github_file = (
870+
f"{DOCKER_COMPOSE_BASE}/{KAFKA_SETUP_QUICKSTART_COMPOSE_FILE}"
871+
)
872+
873+
default_consumer_compose_file = (
874+
Path(DATAHUB_ROOT_FOLDER) / "quickstart/docker-compose.consumers.yml"
875+
)
876+
with open(
877+
default_consumer_compose_file, "wb"
878+
) if default_consumer_compose_file else tempfile.NamedTemporaryFile(
879+
suffix=".yml", delete=False
880+
) as tmp_file:
881+
path = pathlib.Path(tmp_file.name)
882+
quickstart_compose_file_list.append(path)
883+
click.echo(
884+
f"Fetching consumer docker-compose file {kafka_setup_github_file} from GitHub"
885+
)
886+
# Download the quickstart docker-compose file from GitHub.
887+
quickstart_download_response = request_session.get(kafka_setup_github_file)
888+
quickstart_download_response.raise_for_status()
889+
tmp_file.write(quickstart_download_response.content)
890+
logger.debug(f"Copied to {path}")
891+
892+
836893
def valid_restore_options(
837894
restore: bool, restore_indices: bool, no_restore_indices: bool
838895
) -> bool:

0 commit comments

Comments
 (0)
Please sign in to comment.