add kafka test

Author: blacktop

.gitignore

@@ -12,3 +12,4 @@ pcap/test2.pcap
 pcap/test.pcap
 .circleci/build_num
 pcap/local.bro
+kafka.out

Makefile

@@ -29,8 +29,7 @@ ifeq ($(BUILD),elastic)
 	@http localhost:9200/_cat/indices
 	@open -a Safari https://goo.gl/e5v7Qr
 else ifeq ($(BUILD),kafka)
-	@docker-compose -f docker-compose.kafka.yml up -d	bro
-	# @docker-compose -f docker-compose.kafka.yml up consumer
+	@tests/kafka.sh
 else ifeq ($(BUILD),redis)
 	@docker-compose -f docker-compose.redis.yml up -d logstash
 	@docker-compose -f docker-compose.elastic.yml up bro

docker-compose.kafka.yml

@@ -1,9 +1,16 @@
 version: '3'
 services:
+  zookeeper:
+    image: blacktop/kafka:0.11
+    ports:
+      - "2181:2181"
+    command: zookeeper-server-start.sh config/zookeeper.properties
   kafka:
     image: blacktop/kafka:0.11
     ports:
       - "9092:9092"
+    links:
+      - zookeeper
     environment:
       - KAFKA_ADVERTISED_HOST_NAME=localhost
       - KAFKA_CREATE_TOPICS=bro:1:1
@@ -17,11 +24,3 @@ services:
     volumes:
       - ./pcap:/pcap
     command: -F -r heartbleed.pcap local "Site::local_nets += { 192.168.11.0/24 }"
-  consumer:
-    depends_on:
-      - kafka
-    image: blacktop/kafka:0.11
-    links:
-      - kafka:localhost
-    command: |
-      kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic bro --from-beginning

tests/kafka.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# clean up
+rm kafka.out || true
+
+echo "===> Starting kafka..."
+docker run -d --name kafka \
+           -p 9092:9092 \
+           -e KAFKA_ADVERTISED_HOST_NAME=localhost \
+           -e KAFKA_CREATE_TOPICS=bro:1:1 \
+           blacktop/kafka:0.11
+
+echo "===> Starting bro..."
+docker run -d --rm \
+           --link kafka:localhost \
+           -v `pwd`/pcap:/pcap \
+           blacktop/bro:kafka -F -r heartbleed.pcap local "Site::local_nets += { 192.168.11.0/24 }"
+
+sleep 10; echo "===> Starting kafka consumer..."
+kafka-console-consumer -brokers=localhost:9092 -topic bro -offset=oldest > kafka.out &
+sleep 5; kill %1 > /dev/null 2>&1
+
+cat kafka.out | grep 'Value:' | cut -d ':' -f 2- | jq 'select(.notice != null) | .notice.note'