Merge pull request #15 from TOoSmOotH/master

1.0.3

README.md

@@ -1,4 +1,4 @@
-# Security Onion Hybrid Hunter Tech Preview 1.0.2
+# Security Onion Hybrid Hunter Tech Preview 1.0.3
 
 ### About
 Hybrid Hunter is a brand new Security Onion platform with the following characteristics:

VERSION

@@ -1 +1 @@
-1.0.2
+1.0.3

exclude-list.txt

@@ -1,3 +1,2 @@
 salt/bro/files/local.bro
 salt/bro/files/local.bro.community
-salt/suricata/suricata.yaml

salt/bro/init.sls

@@ -65,7 +65,7 @@ localbrosync:
 
 so-bro:
   docker_container.running:
-    - image: toosmooth/so-communitybro:techpreview
+    - image: soshybridhunter/so-communitybro:HH1.0.3
     - privileged: True
     - binds:
       - /nsm/bro/logs:/nsm/bro/logs:rw
@@ -76,6 +76,10 @@ so-bro:
       - /opt/so/conf/bro/policy/custom:/opt/bro/share/bro/policy/custom:ro
       - /opt/so/conf/bro/policy/intel:/opt/bro/share/bro/policy/intel:rw
     - network_mode: host
+    - watch:
+      - file: /opt/so/conf/bro/local.bro
+      - file: /opt/so/conf/bro/node.cfg
+      - file: /opt/so/conf/bro/policy
 
 {% else %}
 localbrosync:
@@ -88,7 +92,7 @@ localbrosync:
 
 so-bro:
   docker_container.running:
-    - image: toosmooth/so-bro:techpreview
+    - image: soshybridhunter/so-bro:HH1.0.3
     - privileged: True
     - binds:
       - /nsm/bro/logs:/nsm/bro/logs:rw
@@ -99,5 +103,10 @@ so-bro:
       - /opt/so/conf/bro/policy/custom:/opt/bro/share/bro/policy/custom:ro
       - /opt/so/conf/bro/policy/intel:/opt/bro/share/bro/policy/intel:rw
     - network_mode: host
+    - watch:
+      - file: /opt/so/conf/bro/local.bro
+      - file: /opt/so/conf/bro/node.cfg
+      - file: /opt/so/conf/bro/policy
+
 
 {% endif %}

salt/common/init.sls

@@ -112,7 +112,7 @@ nginxtmp:
 # Start the core docker
 so-core:
   docker_container.running:
-    - image: toosmooth/so-core:techpreview
+    - image: soshybridhunter/so-core:HH1.0.3
     - hostname: so-core
     - user: socore
     - binds:
@@ -127,3 +127,5 @@ so-core:
     - port_bindings:
       - 80:80
       - 443:443
+    - watch:
+      - file: /opt/so/conf/nginx/nginx.conf

salt/elasticsearch/init.sls

@@ -95,9 +95,9 @@ eslogdir:
 
 so-elasticsearch:
   docker_container.running:
-    - image: securityonionsolutions/so-elasticsearch:latest
+    - image: soshybridhunter/so-elasticsearch:HH1.0.3
     - hostname: elasticsearch
-    - name: elasticsearch
+    - name: so-elasticsearch
     - user: elasticsearch
     - environment:
       - bootstrap.memory_lock=true
@@ -148,7 +148,7 @@ freqlogdir:
 
 so-freq:
   docker_container.running:
-    - image: securityonionsolutions/so-freqserver
+    - image: soshybridhunter/so-freqserver:HH1.0.3
     - hostname: freqserver
     - user: freqserver
     - binds:
@@ -183,7 +183,7 @@ dstatslogdir:
 
 so-domainstats:
   docker_container.running:
-    - image: securityonionsolutions/so-domainstats
+    - image: soshybridhunter/so-domainstats:HH1.0.3
     - hostname: domainstats
     - name: domainstats
     - user: domainstats
@@ -248,7 +248,7 @@ curconf:
 
 so-curator:
   docker_container.running:
-    - image: securityonionsolutions/so-curator
+    - image: soshybridhunter/so-curator:HH1.0.3
     - hostname: curator
     - name: curator
     - user: curator
@@ -309,7 +309,7 @@ elastaconf:
 
 so-elastalert:
   docker_container.running:
-    - image: securityonionsolutions/so-elastalert
+    - image: soshybridhunter/so-elastalert:HH1.0.3
     - hostname: elastalert
     - name: elastalert
     - user: elastalert

salt/filebeat/etc/filebeat.yml

@@ -1,5 +1,6 @@
 {%- set MASTER = grains['master'] %}
 {%- set HOSTNAME = salt['grains.get']('host', '') %}
+{%- set BROVER = salt['pillar.get']('static:broversion', 'COMMUNITY') %}
 
 name: {{ HOSTNAME }}
 
@@ -10,7 +11,7 @@ filebeat.modules:
 # List of prospectors to fetch data.
 filebeat.prospectors:
 #------------------------------ Log prospector --------------------------------
-
+{%- if BROVER != 'SURICATA' %}
 {%- for LOGNAME in salt['pillar.get']('brologs:enabled', '')  %}
   - type: log
     paths:
@@ -23,6 +24,7 @@ filebeat.prospectors:
     close_removed: false
 
 {%- endfor %}
+{%- endif %}
 
   - type: log
     paths:

salt/filebeat/init.sls

@@ -52,7 +52,7 @@ filebeatconfsync:
 
 so-filebeat:
   docker_container.running:
-    - image: toosmooth/so-filebeat:techpreview
+    - image: soshybridhunter/so-filebeat:HH1.0.3
     - hostname: so-filebeat
     - user: root
     - extra_hosts: {{ MASTER }}:{{ MASTERIP }}
@@ -64,3 +64,5 @@ so-filebeat:
       - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro
       - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro
       - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro
+    - watch:
+      - file: /opt/so/conf/filebeat/etc

salt/idstools/init.sls

@@ -53,10 +53,9 @@ toosmooth/so-idstools:test2:
 
 so-idstools:
   docker_container.running:
-    - image: toosmooth/so-idstools:test2
+    - image: soshybridhunter/so-idstools:HH1.0.3
     - hostname: so-idstools
     - user: socore
     - binds:
       - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro
       - /opt/so/rules/nids:/opt/so/rules/nids:rw
-

salt/kibana/init.sls

@@ -59,7 +59,7 @@ synckibanacustom:
 # Start the kibana docker
 so-kibana:
   docker_container.running:
-    - image: toosmooth/so-kibana:techpreview
+    - image: soshybridhunter/so-kibana:HH1.0.3
     - hostname: kibana
     - user: kibana
     - environment:

salt/logstash/files/custom/templates/Drop.Your.Custom.Templates.Here.conf

@@ -0,0 +1,2 @@
+# Reference /usr/share/logstash/pipeline.custom/templates/YOURTEMPLATE.json
+#

salt/logstash/init.sls

@@ -63,6 +63,20 @@ lscustdir:
     - group: 939
     - makedirs: True
 
+lscustparserdir:
+  file.directory:
+    - name: /opt/so/conf/logstash/custom/parsers
+    - user: 931
+    - group: 939
+    - makedirs: True
+
+lscusttemplatedir:
+  file.directory:
+    - name: /opt/so/conf/logstash/custom/templates
+    - user: 931
+    - group: 939
+    - makedirs: True
+
 # Copy down all the configs including custom - TODO add watch restart
 lssync:
   file.recurse:
@@ -109,7 +123,7 @@ lslogdir:
 
 so-logstash:
   docker_container.running:
-    - image: toosmooth/so-logstash:HH1.0.2
+    - image: soshybridhunter/so-logstash:HH1.0.3
     - hostname: so-logstash
     - name: so-logstash
     - user: logstash
@@ -145,3 +159,5 @@ so-logstash:
       - /nsm/bro:/nsm/bro:ro
       - /opt/so/log/suricata:/suricata:ro
       {%- endif %}
+    - watch:
+      - file: /opt/so/conf/logstash

salt/master/init.sls

@@ -49,7 +49,7 @@ acngcopyconf:
 # Install the apt-cacher-ng container
 so-aptcacherng:
   docker_container.running:
-    - image: toosmooth/so-acng:techpreview
+    - image: soshybridhunter/so-acng:HH1.0.3
     - hostname: so-acng
     - port_bindings:
       - 0.0.0.0:3142:3142

salt/pcap/files/config

@@ -1,4 +1,4 @@
-{%- set interface = salt['pillar.get']('sensor:interface', '') %}
+{%- set interface = salt['pillar.get']('sensor:interface', 'bond0') %}
 {
   "Threads": [
     { "PacketsDirectory": "/nsm/pcap"

salt/pcap/init.sls

@@ -73,7 +73,7 @@ stenolog:
 
 so-steno:
   docker_container.running:
-    - image: toosmooth/so-steno:techpreview
+    - image: soshybridhunter/so-steno:HH1.0.3
     - network_mode: host
     - privileged: True
     - port_bindings:

salt/redis/init.sls

@@ -49,7 +49,7 @@ toosmooth/so-redis:test2:
 
 so-redis:
   docker_container.running:
-    - image: toosmooth/so-redis:test2
+    - image: soshybridhunter/so-redis:HH1.0.3
     - hostname: so-redis
     - user: socore
     - port_bindings:
@@ -59,4 +59,3 @@ so-redis:
       - /opt/so/conf/redis/etc/redis.conf:/usr/local/etc/redis/redis.conf:ro
       - /opt/so/conf/redis/working:/redis:rw
     - entrypoint: "redis-server /usr/local/etc/redis/redis.conf"
-

salt/suricata/init.sls

@@ -14,6 +14,7 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 {% set interface = salt['pillar.get']('sensor:interface', 'bond0') %}
+{%- set BROVER = salt['pillar.get']('static:broversion', 'COMMUNITY') %}
 
 # Suricata
 
@@ -60,14 +61,18 @@ surirulesync:
 suriconfigsync:
   file.managed:
     - name: /opt/so/conf/suricata/suricata.yaml
+    {%- if BROVER != 'SURICATA' %}
     - source: salt://suricata/files/suricata.yaml
+    {%- else %}
+    - source: salt://suricata/files/suricataMETA.yaml
+    {%- endif %}
     - user: 940
     - group: 940
     - template: jinja
 
 so-suricata:
   docker_container.running:
-    - image: toosmooth/so-suricata:techpreview
+    - image: soshybridhunter/so-suricata:HH1.0.3
     - privileged: True
     - environment:
       - INTERFACE={{ interface }}
@@ -76,3 +81,6 @@ so-suricata:
       - /opt/so/conf/suricata/rules:/etc/suricata/rules:ro
       - /opt/so/log/suricata/:/var/log/suricata/:rw
     - network_mode: host
+    - watch:
+      - file: /opt/so/conf/suricata/suricata.yaml
+      - file: /opt/so/conf/suricata/rules/

salt/top.sls

@@ -1,3 +1,4 @@
+{%- set BROVER = salt['pillar.get']('static:broversion', 'COMMUNITY') %}
 base:
   'G@role:so-sensor':
     - ssl
@@ -5,7 +6,9 @@ base:
     - firewall
     - pcap
     - suricata
+    {%- if BROVER != 'SURICATA' %}
     - bro
+    {%- endif %}
     - filebeat
 
   'G@role:so-eval':

so-setup-network.sh

@@ -35,8 +35,10 @@ accept_salt_key_local() {
 
 accept_salt_key_remote() {
 
-  # Accept the key remotely so the device can check in
+  # Delete the key just in case.
-  ssh -v -i /root/.ssh/so.key socore@$MSRV sudo salt-key -a $HOSTNAME -y
+  ssh -i /root/.ssh/so.key socore@$MSRV sudo salt-key -d $HOSTNAME -y
+  salt-call state.apply ca
+  ssh -i /root/.ssh/so.key socore@$MSRV sudo salt-key -a $HOSTNAME -y
 
 }
 
@@ -156,13 +158,13 @@ create_bond() {
   if [ $OS == 'centos' ]; then
     modprobe --first-time bonding
     touch /etc/sysconfig/network-scripts/ifcfg-bond0
-    echo "DEVICE=bond0" >> /etc/sysconfig/network-scripts/ifcfg-bond0
+    echo "DEVICE=bond0" > /etc/sysconfig/network-scripts/ifcfg-bond0
     echo "NAME=bond0" >> /etc/sysconfig/network-scripts/ifcfg-bond0
     echo "Type=Bond" >> /etc/sysconfig/network-scripts/ifcfg-bond0
     echo "BONDING_MASTER=yes" >> /etc/sysconfig/network-scripts/ifcfg-bond0
     echo "BOOTPROTO=none" >> /etc/sysconfig/network-scripts/ifcfg-bond0
     echo "BONDING_OPTS=\"mode=0\"" >> /etc/sysconfig/network-scripts/ifcfg-bond0
-    echo "ONBOOT=yes"
+    echo "ONBOOT=yes" >> /etc/sysconfig/network-scripts/ifcfg-bond0
 
     # Create Bond configs for the selected monitor interface
     for BNIC in ${BNICS[@]}; do
@@ -208,7 +210,6 @@ create_bond() {
     for BNIC in ${BNICS[@]}; do
 
       BNIC=$(echo $BNIC |  cut -d\" -f2)
-      echo ""
       echo "auto $BNIC" >> /etc/network/interfaces.d/$BNIC
       echo "iface $BNIC inet manual" >> /etc/network/interfaces.d/$BNIC
       echo "  up ip link set \$IFACE promisc on arp off up" >> /etc/network/interfaces.d/$BNIC
@@ -216,13 +217,12 @@ create_bond() {
       echo "  post-up ethtool -G \$IFACE rx 4096; for i in rx tx sg tso ufo gso gro lro; do ethtool -K \$IFACE \$i off; done" >> /etc/network/interfaces.d/$BNIC
       echo "  post-up echo 1 > /proc/sys/net/ipv6/conf/\$IFACE/disable_ipv6" >> /etc/network/interfaces.d/$BNIC
       echo "  bond-master bond0" >> /etc/network/interfaces.d/$BNIC
-      echo ""
 
     done
 
     BN=("${BNICS[@]//\"/}")
 
-    echo "auto bond0" >> /etc/network/interfaces.d/bond0
+    echo "auto bond0" > /etc/network/interfaces.d/bond0
     echo "iface bond0 inet manual" >> /etc/network/interfaces.d/bond0
     echo "  bond-mode 0" >> /etc/network/interfaces.d/bond0
     echo "  bond-slaves $BN" >> /etc/network/interfaces.d/bond0
@@ -240,6 +240,7 @@ detect_os() {
   echo "Detecting Base OS"
   if [ -f /etc/redhat-release ]; then
     OS=centos
+    yum -y install bind-utils
   elif [ -f /etc/os-release ]; then
     OS=ubuntu
   else
@@ -333,15 +334,15 @@ got_root() {
 install_cleanup() {
 
   # Clean up after ourselves
-  rm -rf ./installtmp
+  rm -rf /root/installtmp
 
 }
 
 install_prep() {
 
   # Create a tmp space that isn't in /tmp
-  mkdir ./installtmp
+  mkdir /root/installtmp
-  TMP=./installtmp
+  TMP=/root/installtmp
 
 }
 
@@ -648,12 +649,6 @@ sensor_pillar() {
       SPIN=$(echo $SPIN |  cut -d\" -f2)
     echo "    - $SPIN" >> $TMP/$HOSTNAME.sls
     done
-    #SP=("${SURIPINS[@]//\"/}")
-    #SPINS=${SP// /,}
-    #SCOUNT=${#SURIPINS[@]}
-
-    #echo "  suripins: $SPINS" >> $TMP/$HOSTNAME.sls
-    #echo "  surithreads: $SCOUNT"
   else
     echo "  bro_lbprocs: $BASICBRO" >> $TMP/$HOSTNAME.sls
     echo "  suriprocs: $BASICSURI" >> $TMP/$HOSTNAME.sls
@@ -686,14 +681,14 @@ set_initial_firewall_policy() {
   fi
 
   if [ $INSTALLTYPE == 'SENSORONLY' ]; then
-    ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP
+    ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP
-    ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP
+    ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP
   fi
 
   if [ $INSTALLTYPE == 'STORAGENODE' ]; then
-    ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP
+    ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP
-    ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh storage_nodes $MAINIP
+    ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh storage_nodes $MAINIP
-    ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $HOSTNAME $MAINIP
+    ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $HOSTNAME $MAINIP
   fi
 
   if [ $INSTALLTYPE == 'PARSINGNODE' ]; then
@@ -746,10 +741,14 @@ set_updates() {
 
 update_sudoers() {
 
+  if ! grep -qE '^socore\ ALL=\(ALL\)\ NOPASSWD:(\/usr\/bin\/salt\-key|\/opt\/so\/saltstack)' /etc/sudoers; then
     # Update Sudoers so that socore can accept keys without a password
     echo "socore ALL=(ALL) NOPASSWD:/usr/bin/salt-key" | sudo tee -a /etc/sudoers
     echo "socore ALL=(ALL) NOPASSWD:/opt/so/saltstack/pillar/firewall/addfirewall.sh" | sudo tee -a /etc/sudoers
     echo "socore ALL=(ALL) NOPASSWD:/opt/so/saltstack/pillar/data/addtotab.sh" | sudo tee -a /etc/sudoers
+  else
+    echo "User socore already granted sudo privileges"
+  fi
 
 }
 
@@ -791,7 +790,8 @@ whiptail_bro_pins() {
 
 whiptail_bro_version() {
 
-  BROVERSION=$(whiptail --title "Security Onion Setup" --radiolist "Which version of Bro would you like to use?" 20 78 4 "COMMUNITY" "Install Community Bro" ON "ZEEK" "Install Zeek" OFF 3>&1 1>&2 2>&3)
+  BROVERSION=$(whiptail --title "Security Onion Setup" --radiolist "What tool would you like to use to generate meta data?" 20 78 4 "COMMUNITY" "Install Community Bro" ON \
+   "ZEEK" "Install Zeek" OFF "SURICATA" "SUPER EXPERIMENTAL" OFF 3>&1 1>&2 2>&3)
 
   local exitstatus=$?
   whiptail_check_exitstatus $exitstatus

updatemaster.sh

@@ -3,11 +3,11 @@
 # Clone github
 mkdir /tmp/sogh
 cd /tmp/sogh
-git clone https://github.com/TOoSmOotH/securityonion-saltstack.git
+#git clone https://github.com/TOoSmOotH/securityonion-saltstack.git
+git clone https://github.com/Security-Onion-Solutions/securityonion-saltstack.git
 cd securityonion-saltstack
-rsync -a pillar /opt/so/saltstack/
 rsync -a --exclude-from 'exclude-list.txt' salt /opt/so/saltstack/
-chown -R socore:socore /opt/so
+chown -R socore:socore /opt/so/saltstack/salt
 chmod 755 /opt/so/saltstack/pillar/firewall/addfirewall.sh
 cd ~
 rm -rf /tmp/sogh