Merge branch 'master' into devel

README.md

@@ -13,6 +13,7 @@ ROCK is a collections platform, in the spirit of Network Security Monitoring by
 * A messaging layer (Kafka and Logstash) that provides flexibility in scaling the platform to meet operational needs, as well as providing some degree of data reliability in transit.
 * Reliable data storage and indexing (Elasticsearch) to support rapid retrieval and analysis (Kibana) of the data.
 
+
 ## Features
 
 * Full Packet Capture via Google Stenographer and Docket.
@@ -25,60 +26,18 @@ ROCK is a collections platform, in the spirit of Network Security Monitoring by
 * Data UI and Visualization via Kibana.
 * Security - The system is developed and tested to run with SELinux enabled.
 
-## Approach
-
-The Ansible playbook that drives this build strives not to use any external roles or other dependencies. The reasoning behind this is to make the rock playbook a "one-stop" reference for a manual build. This allows users to use the build process as a guide when doing larger scale production roll outs without having to decipher a labyrinth of dependencies.
-
-Templated config files have comment sections added near key config items with useful info. They don't all have it, but they get added as remembered.
-
-## Usage
-
-### Operating System Deployment
-
-This system is distributed as an [ISO](https://download.rocknsm.io/) and is designed to be deployed as a secure operating system. This is the only supported method for deployment.
-
-### Service Deployment
-
-Following operating system installation, you can customize the service deployment by editing `/etc/rocknsm/rock/config.yml`.
-
-**NOTE:** If this file does not exist, you can create it with the following command:
 
-```
-sudo /opt/rocknsm/rock/bin/generate_defaults.sh
-```
+### Installation and Usage
 
-Once you are happy with the deployment parameters, run the service deployment as follows:
+Please reference our [documentation](https://rocknsm.gitbooks.io/rocknsm-guide/content/) for all ROCK details to include:
 
-```
-sudo /opt/rocknsm/rock/bin/deploy_rock.sh
-```
+- installation
+- configuration
+- deployment
+- troubleshooting
 
-[![asciicast](https://asciinema.org/a/jnwhnl7N02G1bXbkot9zseirl.png)](https://asciinema.org/a/jnwhnl7N02G1bXbkot9zseirl)
-
-### Functions Check:
-```
-# Check to see that the ES cluster says it's green:
-curl -s localhost:9200/_cluster/health | jq '.'
-
-# See how many documents are in the indexes.  The count should be non-zero.
-curl -s localhost:9200/_all/_count | jq '.'
-
-# You can fire some traffic across the sensor at this point to see if it's collecting.
-# NOTE: This requires that you upload your own test PCAP to the box.
-sudo tcpreplay -i [your monitor interface] /path/to/a/test.pcap
-
-# After replaying some traffic, or just waiting a bit, the count should be going up.
-curl -s localhost:9200/_all/_count | jq '.'
-
-# You should have plain text bro logs showing up in /data/bro/logs/current/:
-ls -ltr /data/bro/logs/current/
-
-# Kafkacat is your kafka swiss army knife.  This command will consume the current queue.  You should see a non-zero offset.
-kafkacat -C -b localhost -t bro_raw -e | wc -l
-
-# If you haven't loaded kibana already, it should be running on port 5601.  This just verifies while you're still on the command line.
-sudo netstat -planet | grep node
-```
 
 ## Thanks
 This architecture is made possible by the efforts of an ever-growing list of amazing people. Look around our Github to see the whole list.
+
+<!-- [![asciicast](https://asciinema.org/a/jnwhnl7N02G1bXbkot9zseirl.png)](https://asciinema.org/a/jnwhnl7N02G1bXbkot9zseirl) -->

playbooks/group_vars/all.yml

@@ -27,7 +27,7 @@ elastic_close_interval: 15
 elastic_delete_interval: 60
 kafka_retention: 168
 suricata_retention: 3
-bro_log_retention: 0
+bro_log_retention: 7
 bro_stats_retention: 0
 
 # Feature options - Don't flip these unless you know what you're doing

playbooks/templates/rock_config.yml.j2

@@ -152,7 +152,6 @@ enable_lighttpd: {{ enable_lighttpd }}
 enable_fsf: {{ enable_fsf }}
 enable_filebeat: {{ enable_filebeat }}
 
-
 ###############################################################################
 #                             NEXT STEP: Deployment
 ###############################################################################

roles/common/files/rockctl

@@ -1,7 +1,19 @@
 #!/bin/bash
 
 action=$1
-psprocs=( zookeeper kafka bro suricata filebeat elasticsearch logstash kibana stenographer )
+psprocs=( zookeeper kafka bro suricata filebeat elasticsearch logstash kibana stenographer fsf docket )
+
+function feature_enabled() {
+  if grep -qiE "^with_$1: (true|yes)" /etc/rocknsm/config.yml; then
+    if grep -qiE "^enable_$1: (true|yes)" /etc/rocknsm/config.yml; then
+      return $?
+    else
+      false
+    fi
+  else
+    false
+  fi
+}
 
 function reverse()
 {
@@ -24,12 +36,26 @@ case $action in
     for proc in "${psprocs[@]}"; do
       echo "${proc^^}: stopping..."
       systemctl stop ${proc}
+      # Lets also make sure the service is disabled
+      if [[ $(systemctl is-enabled ${proc}) == 'enabled' ]] && ! $(feature_enabled ${proc}); then
+        echo "${proc^^} is set to disabled in RockNSM config but is currently enabled in systemctl."
+        echo "Disabling ${proc^^} in systemctl..."
+        systemctl disable --quiet ${proc}
+      fi
     done
     ;;
   "start")
     for proc in "${psprocs[@]}"; do
-      echo "${proc^^}: starting..."
-      systemctl start ${proc}
+      if feature_enabled ${proc}; then
+        echo "${proc^^}: starting..."
+        systemctl start ${proc}
+        # Lets also make sure the service is enabled
+        if [[ $(systemctl is-enabled ${proc}) == 'disabled' ]]; then
+          echo "${proc^^} is set to enabled in RockNSM config but is currently disabled in systemctl."
+          echo "Enabling ${proc^^} in systemctl..."
+          systemctl enable ${proc}
+        fi
+      fi
     done
     ;;
   "status")