Merge pull request #15 from TOoSmOotH/master

1.0.3
2025-12-06 09:12:45 +01:00 · 2018-11-14 15:29:12 -05:00
parent 1b9ce046b5 afdefeada6
commit 9dd3c07ab3
24 changed files with 2023 additions and 56 deletions
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# Security Onion Hybrid Hunter Tech Preview 1.0.2
+# Security Onion Hybrid Hunter Tech Preview 1.0.3

 ### About
 Hybrid Hunter is a brand new Security Onion platform with the following characteristics:
--- a/2
+++ b/2
@@ -1 +1 @@
-1.0.2
+1.0.3
--- a/exclude-list.txt
+++ b/exclude-list.txt
@@ -1,3 +1,2 @@
 salt/bro/files/local.bro
 salt/bro/files/local.bro.community
-salt/suricata/suricata.yaml
--- a/salt/bro/init.sls
+++ b/salt/bro/init.sls
@@ -65,7 +65,7 @@ localbrosync:

 so-bro:
  docker_container.running:
-    - image: toosmooth/so-communitybro:techpreview
+    - image: soshybridhunter/so-communitybro:HH1.0.3
    - privileged: True
    - binds:
      - /nsm/bro/logs:/nsm/bro/logs:rw
@@ -76,6 +76,10 @@ so-bro:
      - /opt/so/conf/bro/policy/custom:/opt/bro/share/bro/policy/custom:ro
      - /opt/so/conf/bro/policy/intel:/opt/bro/share/bro/policy/intel:rw
    - network_mode: host
+    - watch:
+      - file: /opt/so/conf/bro/local.bro
+      - file: /opt/so/conf/bro/node.cfg
+      - file: /opt/so/conf/bro/policy

 {% else %}
 localbrosync:
@@ -88,7 +92,7 @@ localbrosync:

 so-bro:
  docker_container.running:
-    - image: toosmooth/so-bro:techpreview
+    - image: soshybridhunter/so-bro:HH1.0.3
    - privileged: True
    - binds:
      - /nsm/bro/logs:/nsm/bro/logs:rw
@@ -99,5 +103,10 @@ so-bro:
      - /opt/so/conf/bro/policy/custom:/opt/bro/share/bro/policy/custom:ro
      - /opt/so/conf/bro/policy/intel:/opt/bro/share/bro/policy/intel:rw
    - network_mode: host
+    - watch:
+      - file: /opt/so/conf/bro/local.bro
+      - file: /opt/so/conf/bro/node.cfg
+      - file: /opt/so/conf/bro/policy
+

 {% endif %}
--- a/salt/common/init.sls
+++ b/salt/common/init.sls
@@ -112,7 +112,7 @@ nginxtmp:
 # Start the core docker
 so-core:
  docker_container.running:
-    - image: toosmooth/so-core:techpreview
+    - image: soshybridhunter/so-core:HH1.0.3
    - hostname: so-core
    - user: socore
    - binds:
@@ -127,3 +127,5 @@ so-core:
    - port_bindings:
      - 80:80
      - 443:443
+    - watch:
+      - file: /opt/so/conf/nginx/nginx.conf
--- a/salt/elasticsearch/init.sls
+++ b/salt/elasticsearch/init.sls
@@ -95,9 +95,9 @@ eslogdir:

 so-elasticsearch:
  docker_container.running:
-    - image: securityonionsolutions/so-elasticsearch:latest
+    - image: soshybridhunter/so-elasticsearch:HH1.0.3
    - hostname: elasticsearch
-    - name: elasticsearch
+    - name: so-elasticsearch
    - user: elasticsearch
    - environment:
      - bootstrap.memory_lock=true
@@ -148,7 +148,7 @@ freqlogdir:

 so-freq:
  docker_container.running:
-    - image: securityonionsolutions/so-freqserver
+    - image: soshybridhunter/so-freqserver:HH1.0.3
    - hostname: freqserver
    - user: freqserver
    - binds:
@@ -183,7 +183,7 @@ dstatslogdir:

 so-domainstats:
  docker_container.running:
-    - image: securityonionsolutions/so-domainstats
+    - image: soshybridhunter/so-domainstats:HH1.0.3
    - hostname: domainstats
    - name: domainstats
    - user: domainstats
@@ -248,7 +248,7 @@ curconf:

 so-curator:
  docker_container.running:
-    - image: securityonionsolutions/so-curator
+    - image: soshybridhunter/so-curator:HH1.0.3
    - hostname: curator
    - name: curator
    - user: curator
@@ -309,7 +309,7 @@ elastaconf:

 so-elastalert:
  docker_container.running:
-    - image: securityonionsolutions/so-elastalert
+    - image: soshybridhunter/so-elastalert:HH1.0.3
    - hostname: elastalert
    - name: elastalert
    - user: elastalert
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -1,5 +1,6 @@
 {%- set MASTER = grains['master'] %}
 {%- set HOSTNAME = salt['grains.get']('host', '') %}
+{%- set BROVER = salt['pillar.get']('static:broversion', 'COMMUNITY') %}

 name: {{ HOSTNAME }}

@@ -10,7 +11,7 @@ filebeat.modules:
 # List of prospectors to fetch data.
 filebeat.prospectors:
 #------------------------------ Log prospector --------------------------------
-
+{%- if BROVER != 'SURICATA' %}
 {%- for LOGNAME in salt['pillar.get']('brologs:enabled', '')  %}
  - type: log
    paths:
@@ -23,6 +24,7 @@ filebeat.prospectors:
    close_removed: false

 {%- endfor %}
+{%- endif %}

  - type: log
    paths:
--- a/salt/filebeat/init.sls
+++ b/salt/filebeat/init.sls
@@ -52,7 +52,7 @@ filebeatconfsync:

 so-filebeat:
  docker_container.running:
-    - image: toosmooth/so-filebeat:techpreview
+    - image: soshybridhunter/so-filebeat:HH1.0.3
    - hostname: so-filebeat
    - user: root
    - extra_hosts: {{ MASTER }}:{{ MASTERIP }}
@@ -64,3 +64,5 @@ so-filebeat:
      - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro
      - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro
      - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro
+    - watch:
+      - file: /opt/so/conf/filebeat/etc
--- a/salt/idstools/init.sls
+++ b/salt/idstools/init.sls
@@ -53,10 +53,9 @@ toosmooth/so-idstools:test2:

 so-idstools:
  docker_container.running:
-    - image: toosmooth/so-idstools:test2
+    - image: soshybridhunter/so-idstools:HH1.0.3
    - hostname: so-idstools
    - user: socore
    - binds:
      - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro
      - /opt/so/rules/nids:/opt/so/rules/nids:rw
-
--- a/salt/kibana/etc/config.json
+++ b/salt/kibana/etc/config.json
@@ -1,7 +1,7 @@
 { "attributes":
 	{
-	  "defaultIndex": "*:logstash-*",
-	  "discover:sampleSize":"10",
+    "defaultIndex": "*:logstash-*",
+		"discover:sampleSize":"10",
 	  "dashboard:defaultDarkTheme":true,
 	  "timepicker:timeDefaults":"{\n  \"from\": \"now-24h\",\n  \"to\": \"now\",\n  \"mode\": \"quick\"\n}"
 	}
--- a/salt/kibana/init.sls
+++ b/salt/kibana/init.sls
@@ -59,7 +59,7 @@ synckibanacustom:
 # Start the kibana docker
 so-kibana:
  docker_container.running:
-    - image: toosmooth/so-kibana:techpreview
+    - image: soshybridhunter/so-kibana:HH1.0.3
    - hostname: kibana
    - user: kibana
    - environment:
--- a/salt/logstash/files/custom/parsers/Drop.Your.Custom.Parsers.Here.conf
+++ b/salt/logstash/files/custom/parsers/Drop.Your.Custom.Parsers.Here.conf
--- a/salt/logstash/files/custom/templates/Drop.Your.Custom.Templates.Here.conf
+++ b/salt/logstash/files/custom/templates/Drop.Your.Custom.Templates.Here.conf
@@ -0,0 +1,2 @@
+# Reference /usr/share/logstash/pipeline.custom/templates/YOURTEMPLATE.json
+#
--- a/salt/logstash/init.sls
+++ b/salt/logstash/init.sls
@@ -63,6 +63,20 @@ lscustdir:
    - group: 939
    - makedirs: True

+lscustparserdir:
+  file.directory:
+    - name: /opt/so/conf/logstash/custom/parsers
+    - user: 931
+    - group: 939
+    - makedirs: True
+
+lscusttemplatedir:
+  file.directory:
+    - name: /opt/so/conf/logstash/custom/templates
+    - user: 931
+    - group: 939
+    - makedirs: True
+
 # Copy down all the configs including custom - TODO add watch restart
 lssync:
  file.recurse:
@@ -109,7 +123,7 @@ lslogdir:

 so-logstash:
  docker_container.running:
-    - image: toosmooth/so-logstash:HH1.0.2
+    - image: soshybridhunter/so-logstash:HH1.0.3
    - hostname: so-logstash
    - name: so-logstash
    - user: logstash
@@ -145,3 +159,5 @@ so-logstash:
      - /nsm/bro:/nsm/bro:ro
      - /opt/so/log/suricata:/suricata:ro
      {%- endif %}
+    - watch:
+      - file: /opt/so/conf/logstash
--- a/salt/master/init.sls
+++ b/salt/master/init.sls
@@ -49,7 +49,7 @@ acngcopyconf:
 # Install the apt-cacher-ng container
 so-aptcacherng:
  docker_container.running:
-    - image: toosmooth/so-acng:techpreview
+    - image: soshybridhunter/so-acng:HH1.0.3
    - hostname: so-acng
    - port_bindings:
      - 0.0.0.0:3142:3142
--- a/salt/pcap/files/config
+++ b/salt/pcap/files/config
@@ -1,4 +1,4 @@
-{%- set interface = salt['pillar.get']('sensor:interface', '') %}
+{%- set interface = salt['pillar.get']('sensor:interface', 'bond0') %}
 {
  "Threads": [
    { "PacketsDirectory": "/nsm/pcap"
--- a/salt/pcap/init.sls
+++ b/salt/pcap/init.sls
@@ -73,7 +73,7 @@ stenolog:

 so-steno:
  docker_container.running:
-    - image: toosmooth/so-steno:techpreview
+    - image: soshybridhunter/so-steno:HH1.0.3
    - network_mode: host
    - privileged: True
    - port_bindings:
--- a/salt/redis/init.sls
+++ b/salt/redis/init.sls
@@ -49,7 +49,7 @@ toosmooth/so-redis:test2:

 so-redis:
  docker_container.running:
-    - image: toosmooth/so-redis:test2
+    - image: soshybridhunter/so-redis:HH1.0.3
    - hostname: so-redis
    - user: socore
    - port_bindings:
@@ -59,4 +59,3 @@ so-redis:
      - /opt/so/conf/redis/etc/redis.conf:/usr/local/etc/redis/redis.conf:ro
      - /opt/so/conf/redis/working:/redis:rw
    - entrypoint: "redis-server /usr/local/etc/redis/redis.conf"
-
--- a/salt/suricata/files/suricataMETA.yaml
+++ b/salt/suricata/files/suricataMETA.yaml
--- a/salt/suricata/init.sls
+++ b/salt/suricata/init.sls
@@ -14,6 +14,7 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.

 {% set interface = salt['pillar.get']('sensor:interface', 'bond0') %}
+{%- set BROVER = salt['pillar.get']('static:broversion', 'COMMUNITY') %}

 # Suricata

@@ -60,14 +61,18 @@ surirulesync:
 suriconfigsync:
  file.managed:
    - name: /opt/so/conf/suricata/suricata.yaml
+    {%- if BROVER != 'SURICATA' %}
    - source: salt://suricata/files/suricata.yaml
+    {%- else %}
+    - source: salt://suricata/files/suricataMETA.yaml
+    {%- endif %}
    - user: 940
    - group: 940
    - template: jinja

 so-suricata:
  docker_container.running:
-    - image: toosmooth/so-suricata:techpreview
+    - image: soshybridhunter/so-suricata:HH1.0.3
    - privileged: True
    - environment:
      - INTERFACE={{ interface }}
@@ -76,3 +81,6 @@ so-suricata:
      - /opt/so/conf/suricata/rules:/etc/suricata/rules:ro
      - /opt/so/log/suricata/:/var/log/suricata/:rw
    - network_mode: host
+    - watch:
+      - file: /opt/so/conf/suricata/suricata.yaml
+      - file: /opt/so/conf/suricata/rules/
--- a/salt/top.sls
+++ b/salt/top.sls
@@ -1,3 +1,4 @@
+{%- set BROVER = salt['pillar.get']('static:broversion', 'COMMUNITY') %}
 base:
  'G@role:so-sensor':
    - ssl
@@ -5,7 +6,9 @@ base:
    - firewall
    - pcap
    - suricata
+    {%- if BROVER != 'SURICATA' %}
    - bro
+    {%- endif %}
    - filebeat

  'G@role:so-eval':
--- a/so-setup-network.sh
+++ b/so-setup-network.sh
@@ -35,8 +35,10 @@ accept_salt_key_local() {

 accept_salt_key_remote() {

-  # Accept the key remotely so the device can check in
-  ssh -v -i /root/.ssh/so.key socore@$MSRV sudo salt-key -a $HOSTNAME -y
+  # Delete the key just in case.
+  ssh -i /root/.ssh/so.key socore@$MSRV sudo salt-key -d $HOSTNAME -y
+  salt-call state.apply ca
+  ssh -i /root/.ssh/so.key socore@$MSRV sudo salt-key -a $HOSTNAME -y

 }

@@ -156,13 +158,13 @@ create_bond() {
  if [ $OS == 'centos' ]; then
    modprobe --first-time bonding
    touch /etc/sysconfig/network-scripts/ifcfg-bond0
-    echo "DEVICE=bond0" >> /etc/sysconfig/network-scripts/ifcfg-bond0
+    echo "DEVICE=bond0" > /etc/sysconfig/network-scripts/ifcfg-bond0
    echo "NAME=bond0" >> /etc/sysconfig/network-scripts/ifcfg-bond0
    echo "Type=Bond" >> /etc/sysconfig/network-scripts/ifcfg-bond0
    echo "BONDING_MASTER=yes" >> /etc/sysconfig/network-scripts/ifcfg-bond0
    echo "BOOTPROTO=none" >> /etc/sysconfig/network-scripts/ifcfg-bond0
    echo "BONDING_OPTS=\"mode=0\"" >> /etc/sysconfig/network-scripts/ifcfg-bond0
-    echo "ONBOOT=yes"
+    echo "ONBOOT=yes" >> /etc/sysconfig/network-scripts/ifcfg-bond0

    # Create Bond configs for the selected monitor interface
    for BNIC in ${BNICS[@]}; do
@@ -208,7 +210,6 @@ create_bond() {
    for BNIC in ${BNICS[@]}; do

      BNIC=$(echo $BNIC |  cut -d\" -f2)
-      echo ""
      echo "auto $BNIC" >> /etc/network/interfaces.d/$BNIC
      echo "iface $BNIC inet manual" >> /etc/network/interfaces.d/$BNIC
      echo "  up ip link set \$IFACE promisc on arp off up" >> /etc/network/interfaces.d/$BNIC
@@ -216,13 +217,12 @@ create_bond() {
      echo "  post-up ethtool -G \$IFACE rx 4096; for i in rx tx sg tso ufo gso gro lro; do ethtool -K \$IFACE \$i off; done" >> /etc/network/interfaces.d/$BNIC
      echo "  post-up echo 1 > /proc/sys/net/ipv6/conf/\$IFACE/disable_ipv6" >> /etc/network/interfaces.d/$BNIC
      echo "  bond-master bond0" >> /etc/network/interfaces.d/$BNIC
-      echo ""

    done

    BN=("${BNICS[@]//\"/}")

-    echo "auto bond0" >> /etc/network/interfaces.d/bond0
+    echo "auto bond0" > /etc/network/interfaces.d/bond0
    echo "iface bond0 inet manual" >> /etc/network/interfaces.d/bond0
    echo "  bond-mode 0" >> /etc/network/interfaces.d/bond0
    echo "  bond-slaves $BN" >> /etc/network/interfaces.d/bond0
@@ -240,6 +240,7 @@ detect_os() {
  echo "Detecting Base OS"
  if [ -f /etc/redhat-release ]; then
    OS=centos
+    yum -y install bind-utils
  elif [ -f /etc/os-release ]; then
    OS=ubuntu
  else
@@ -333,15 +334,15 @@ got_root() {
 install_cleanup() {

  # Clean up after ourselves
-  rm -rf ./installtmp
+  rm -rf /root/installtmp

 }

 install_prep() {

  # Create a tmp space that isn't in /tmp
-  mkdir ./installtmp
-  TMP=./installtmp
+  mkdir /root/installtmp
+  TMP=/root/installtmp

 }

@@ -648,12 +649,6 @@ sensor_pillar() {
      SPIN=$(echo $SPIN |  cut -d\" -f2)
    echo "    - $SPIN" >> $TMP/$HOSTNAME.sls
    done
-    #SP=("${SURIPINS[@]//\"/}")
-    #SPINS=${SP// /,}
-    #SCOUNT=${#SURIPINS[@]}
-
-    #echo "  suripins: $SPINS" >> $TMP/$HOSTNAME.sls
-    #echo "  surithreads: $SCOUNT"
  else
    echo "  bro_lbprocs: $BASICBRO" >> $TMP/$HOSTNAME.sls
    echo "  suriprocs: $BASICSURI" >> $TMP/$HOSTNAME.sls
@@ -686,14 +681,14 @@ set_initial_firewall_policy() {
  fi

  if [ $INSTALLTYPE == 'SENSORONLY' ]; then
-    ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP
-    ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP
+    ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP
+    ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP
  fi

  if [ $INSTALLTYPE == 'STORAGENODE' ]; then
-    ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP
-    ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh storage_nodes $MAINIP
-    ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $HOSTNAME $MAINIP
+    ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP
+    ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh storage_nodes $MAINIP
+    ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $HOSTNAME $MAINIP
  fi

  if [ $INSTALLTYPE == 'PARSINGNODE' ]; then
@@ -746,10 +741,14 @@ set_updates() {

 update_sudoers() {

-  # Update Sudoers so that socore can accept keys without a password
-  echo "socore ALL=(ALL) NOPASSWD:/usr/bin/salt-key" | sudo tee -a /etc/sudoers
-  echo "socore ALL=(ALL) NOPASSWD:/opt/so/saltstack/pillar/firewall/addfirewall.sh" | sudo tee -a /etc/sudoers
-  echo "socore ALL=(ALL) NOPASSWD:/opt/so/saltstack/pillar/data/addtotab.sh" | sudo tee -a /etc/sudoers
+  if ! grep -qE '^socore\ ALL=\(ALL\)\ NOPASSWD:(\/usr\/bin\/salt\-key|\/opt\/so\/saltstack)' /etc/sudoers; then
+    # Update Sudoers so that socore can accept keys without a password
+    echo "socore ALL=(ALL) NOPASSWD:/usr/bin/salt-key" | sudo tee -a /etc/sudoers
+    echo "socore ALL=(ALL) NOPASSWD:/opt/so/saltstack/pillar/firewall/addfirewall.sh" | sudo tee -a /etc/sudoers
+    echo "socore ALL=(ALL) NOPASSWD:/opt/so/saltstack/pillar/data/addtotab.sh" | sudo tee -a /etc/sudoers
+  else
+    echo "User socore already granted sudo privileges"
+  fi

 }

@@ -791,7 +790,8 @@ whiptail_bro_pins() {

 whiptail_bro_version() {

-  BROVERSION=$(whiptail --title "Security Onion Setup" --radiolist "Which version of Bro would you like to use?" 20 78 4 "COMMUNITY" "Install Community Bro" ON "ZEEK" "Install Zeek" OFF 3>&1 1>&2 2>&3)
+  BROVERSION=$(whiptail --title "Security Onion Setup" --radiolist "What tool would you like to use to generate meta data?" 20 78 4 "COMMUNITY" "Install Community Bro" ON \
+   "ZEEK" "Install Zeek" OFF "SURICATA" "SUPER EXPERIMENTAL" OFF 3>&1 1>&2 2>&3)

  local exitstatus=$?
  whiptail_check_exitstatus $exitstatus
--- a/updatemaster.sh
+++ b/updatemaster.sh
@@ -3,11 +3,11 @@
 # Clone github
 mkdir /tmp/sogh
 cd /tmp/sogh
-git clone https://github.com/TOoSmOotH/securityonion-saltstack.git
+#git clone https://github.com/TOoSmOotH/securityonion-saltstack.git
+git clone https://github.com/Security-Onion-Solutions/securityonion-saltstack.git
 cd securityonion-saltstack
-rsync -a pillar /opt/so/saltstack/
 rsync -a --exclude-from 'exclude-list.txt' salt /opt/so/saltstack/
-chown -R socore:socore /opt/so
+chown -R socore:socore /opt/so/saltstack/salt
 chmod 755 /opt/so/saltstack/pillar/firewall/addfirewall.sh
 cd ~
 rm -rf /tmp/sogh
@@ -1 +1 @@
 .0.2
 .0.3