Merge branch 'master' of https://github.com/TOoSmOotH/securityonion-saltstack

salt/common/init.sls

@@ -141,6 +141,8 @@ so-core:
     - watch:
       - file: /opt/so/conf/nginx/nginx.conf
 
+# If master or eval, install Grafana/Telegraf/Influx
+{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' and GRAFANA == 1 %}
 # Add Telegraf to monitor all the things.
 tgraflogdir:
   file.directory:
@@ -213,9 +215,6 @@ so-telegraf:
       - /opt/so/conf/telegraf/etc/telegraf.conf
       - /opt/so/conf/telegraf/scripts
 
-# If its a master or eval lets install the back end for now
-{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' and GRAFANA == 1 %}
-
 # Influx DB
 influxconfdir:
   file.directory:

salt/common/nginx/nginx.conf.so-eval

@@ -188,6 +188,18 @@ http {
           proxy_set_header      Proxy "";
 
         }
+        
+        location /cyberchef/ {
+          proxy_pass http://{{ masterip }}:9080/;
+          proxy_read_timeout    90;
+          proxy_connect_timeout 90;
+          proxy_http_version 1.1; # this is essential for chunked responses to work
+          proxy_set_header      Host $host;
+          proxy_set_header      X-Real-IP $remote_addr;
+          proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header      Proxy "";
+
+        }
 	
         location /soctopus/ {
           proxy_pass http://{{ masterip }}:7000/;

salt/common/nginx/nginx.conf.so-master

@@ -187,6 +187,18 @@ http {
           proxy_set_header      Proxy "";
 
         }
+
+        location /cyberchef/ {
+          proxy_pass http://{{ masterip }}:9080/;
+          proxy_read_timeout    90;
+          proxy_connect_timeout 90;
+          proxy_http_version 1.1; # this is essential for chunked responses to work
+          proxy_set_header      Host $host;
+          proxy_set_header      X-Real-IP $remote_addr;
+          proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header      Proxy "";
+
+        }
 	
         location /soctopus/ {
           proxy_pass http://{{ masterip }}:7000/;

salt/cyberchef/init.sls

@@ -0,0 +1,53 @@
+# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Create the cyberchef group
+cyberchefgroup:
+  group.present:
+    - name: cyberchef
+    - gid: 946
+
+# Add the cyberchef user
+cyberchef:
+  user.present:
+    - uid: 946
+    - gid: 946
+    - home: /opt/so/conf/cyberchef
+
+cyberchefconfdir:
+  file.directory:
+    - name: /opt/so/conf/cyberchef
+    - user: 946
+    - group: 939
+    - makedirs: True
+
+cybercheflog:
+  file.directory:
+    - name: /opt/so/log/cyberchef
+    - user: 946
+    - group: 946
+    - makedirs: True
+
+so-cyberchefimage:
+ cmd.run:
+   - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-cyberchef:HH1.1.3
+
+so-cyberchef:
+  docker_container.running:
+    - require:
+      - so-cyberchefimage
+    - image: docker.io/soshybridhunter/so-cyberchef:HH1.1.3
+    - port_bindings:
+      - 0.0.0.0:9080:8080

salt/firewall/init.sls

@@ -276,6 +276,18 @@ enable_master_cortex_9001_{{ip}}:
     - position: 1
     - save: True 
 
+enable_master_cyberchef_9080_{{ip}}:
+  iptables.insert:
+    - table: filter
+    - chain: DOCKER-USER
+    - jump: ACCEPT
+    - proto: tcp
+    - source: {{ ip }}
+    - dport: 9080
+    - position: 1
+    - save: True
+
+
 {% endfor %}
 
 # Make it so all the minions can talk to salt and update etc.

salt/fleet/init.sls

@@ -61,13 +61,13 @@ fleetdbpriv:
 
 so-fleetimage:
  cmd.run:
-   - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-fleet:HH1.1.0
+   - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-fleet:HH1.1.3
 
 so-fleet:
   docker_container.running:
     - require:
       - so-fleetimage
-    - image: docker.io/soshybridhunter/so-fleet:HH1.1.0
+    - image: docker.io/soshybridhunter/so-fleet:HH1.1.3
     - hostname: so-fleet
     - port_bindings:
       - 0.0.0.0:8080:8080
@@ -83,6 +83,7 @@ so-fleet:
       - KOLIDE_AUTH_JWT_KEY=thisisatest
       - KOLIDE_OSQUERY_STATUS_LOG_FILE=/var/log/osquery/status.log
       - KOLIDE_OSQUERY_RESULT_LOG_FILE=/var/log/osquery/result.log
+      - KOLIDE_SERVER_URL_PREFIX=/fleet
     - binds:
       - /etc/pki/fleet.key:/ssl/server.key:ro
       - /etc/pki/fleet.crt:/ssl/server.cert:ro

salt/playbook/init.sls

@@ -26,13 +26,13 @@ navigatorconfig:
 
 so-playbookimage:
   cmd.run:
-    - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-playbook:HH1.1.1
+    - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-playbook:HH1.1.3
 
 so-playbook:
   docker_container.running:
     - require:
       - so-playbookimage
-    - image: docker.io/soshybridhunter/so-playbook:HH1.1.1
+    - image: docker.io/soshybridhunter/so-playbook:HH1.1.3
     - hostname: playbook
     - name: so-playbook
     - binds:

salt/soctopus/files/templates/generic.template

@@ -1,23 +1,6 @@
 {% set es = salt['pillar.get']('static:masterip', '') %}
 {% set hivehost = salt['pillar.get']('static:masterip', '') %}
 {% set hivekey = salt['pillar.get']('static:hivekey', '') %}
-es_host: {{es}}
-es_port: 9200
-name: Alert-Name
-type: frequency
-index: "*:logstash-*"
-num_events: 1
-timeframe:
-    minutes: 10
-buffer_time:
-    minutes: 10
-allow_buffer_time_overlap: true
-
-filter:
-- query:
-   query_string:
-      query: 'select from test'
-
 alert: modules.so.thehive.TheHiveAlerter
 
 hive_connection:
@@ -30,11 +13,11 @@ hive_proxies:
 
 hive_alert_config:
   title: '{rule[name]}'
-  type: 'external'
+  type: 'playbook'
   source: 'SecurityOnion'
-  description: '`Data:` {match[message]}'
+  description: "`Play:` https://{{es}}/playbook/issues/6000 \n\n `View Event:` <https://{{es}}/kibana/app/kibana#/discover?_g=()&_a=(columns:!(_source),interval:auto,query:(language:lucene,query:'_id:{match[_id]}'),sort:!('@timestamp',desc))>  \n\n `Raw Data:` {match[message]}"
   severity: 2
-  tags: ['elastalert', 'SecurityOnion']
+  tags: ['playbook']
   tlp: 3
   status: 'New'
   follow: True

salt/soctopus/files/templates/osquery.template

@@ -1,23 +1,6 @@
 {% set es = salt['pillar.get']('static:masterip', '') %}
 {% set hivehost = salt['pillar.get']('static:masterip', '') %}
 {% set hivekey = salt['pillar.get']('static:hivekey', '') %}
-es_host: {{es}}
-es_port: 9200
-name: Alert-Name
-type: frequency
-index: "*:logstash-*"
-num_events: 1
-timeframe:
-    minutes: 10
-buffer_time:
-    minutes: 10
-allow_buffer_time_overlap: true
-
-filter:
-- query:
-   query_string:
-      query: 'select from test'
-
 alert: modules.so.thehive.TheHiveAlerter
 
 hive_connection:
@@ -28,20 +11,22 @@ hive_proxies:
   http: ''
   https: ''
 
-hive_alert_config:
-  title: '{rule[name]} -- {match[osquery][hostname]} -- {match[osquery][name]}'
-  type: 'external'
-  source: 'SecurityOnion'
-  description: '`Hostname:` __{match[osquery][hostname]}__  `Live Query:`__[Pivot Link](https://{{es}}/fleet/queries/new?host_uuids={match[osquery][LiveQuery]})__ `Pack:`  __{match[osquery][name]}__ `Data:` {match[osquery][columns]}'
-  severity: 2
-  tags: ['elastalert', 'SecurityOnion']
-  tlp: 3
-  status: 'New'
-  follow: True
-  caseTemplate: '5000'
-
 hive_observable_data_mapping:
   - ip: '{match[osquery][EndpointIP1]}'
   - ip: '{match[osquery][EndpointIP2]}'
   - other: '{match[osquery][hostIdentifier]}'
   - other: '{match[osquery][hostname]}'
+
+hive_alert_config:
+  title: '{rule[name]} -- {match[osquery][hostname]} -- {match[osquery][name]}'
+  type: 'osquery'
+  source: 'SecurityOnion'
+  description: "`Play:` https://{{es}}/playbook/issues/6000 \n\n `View Event:` <https://{{es}}/kibana/app/kibana#/discover?_g=()&_a=(columns:!(_source),interval:auto,query:(language:lucene,query:'_id:{match[_id]}'),sort:!('@timestamp',desc))>  \n\n `Hostname:` __{match[osquery][hostname]}__  `Live Query:`__[Pivot Link](https://{{es}}/fleet/queries/new?host_uuids={match[osquery][LiveQuery]})__ `Pack:`  __{match[osquery][name]}__ `Data:` {match[osquery][columns]}"
+  severity: 2
+  tags: ['playbook','osquery']
+  tlp: 3
+  status: 'New'
+  follow: True
+  caseTemplate: '5000'
+  
+ 

salt/soctopus/init.sls

@@ -12,7 +12,7 @@ soctopussync:
     - user: 939
     - group: 939
     - template: jinja
- 
+
 soctopuslogdir:
   file.directory:
     - name: /opt/so/log/soctopus
@@ -46,18 +46,18 @@ navigatordefaultlayer:
 
 so-soctopusimage:
   cmd.run:
-    - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-soctopus:HH1.1.1
+    - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-soctopus:HH1.1.3
 
 so-soctopus:
   docker_container.running:
     - require:
       - so-soctopusimage
-    - image: docker.io/soshybridhunter/so-soctopus:HH1.1.1
+    - image: docker.io/soshybridhunter/so-soctopus:HH1.1.3
     - hostname: soctopus
     - name: so-soctopus
     - binds:
       - /opt/so/conf/soctopus/SOCtopus.conf:/SOCtopus/SOCtopus.conf:ro
-      - /opt/so/log/soctopus/:/var/log/SOCtopus/:rw      
+      - /opt/so/log/soctopus/:/var/log/SOCtopus/:rw
       - /opt/so/rules/elastalert/playbook:/etc/playbook-rules:rw
       - /opt/so/conf/playbook/nav_layer_playbook.json:/etc/playbook/nav_layer_playbook.json:rw
     - port_bindings:

salt/top.sls

@@ -44,6 +44,7 @@ base:
     - suricata
     - bro
     - curator
+    - cyberchef
     - elastalert
     {%- if OSQUERY != 0 %}
     - fleet
@@ -70,6 +71,7 @@ base:
     - ca
     - ssl
     - common
+    - cyberchef
     - sensoroni
     - firewall
     - master

so-setup-network.sh

@@ -355,12 +355,12 @@ docker_install() {
     yum -y install yum-utils device-mapper-persistent-data lvm2 openssl
     yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
     yum -y update
-    yum -y install docker-ce
+    yum -y install docker-ce python36-docker
     if [ $INSTALLTYPE != 'EVALMODE'  ]; then
       docker_registry
     fi
-    echo "Using pip3 to install docker-py for salt"
+    #echo "Using pip3 to install docker-py for salt"
-    pip3 install -t /usr/lib/python3.6/site-packages/ docker
+    #pip3 install -t /usr/lib/python3.6/site-packages/ docker
     echo "Restarting Docker" >> $SETUPLOG 2>&1
     systemctl restart docker
     systemctl enable docker
@@ -493,7 +493,8 @@ install_pip3() {
   if [ $OS == 'ubuntu' ]; then
     apt-get -y install python3-pip gcc python3-dev
   elif [ $OS == 'centos' ]; then
-    yum -y install python3-pip gcc python3-devel
+    #yum -y install python3-pip gcc python3-devel
+    yum -y install epel-release python3
   fi
 
 }
@@ -510,7 +511,7 @@ install_master() {
 
   # Install the salt master package
   if [ $OS == 'centos' ]; then
-    yum -y install wget salt-common salt-master >> $SETUPLOG 2>&1
+    yum -y install wget salt-common salt-master python36-mysql python36-dateutil python36-m2crypto >> $SETUPLOG 2>&1
 
     # Create a place for the keys for Ubuntu minions
     mkdir -p /opt/so/gpg
@@ -868,8 +869,8 @@ EOF
       fi
     fi
 
-    echo "Using pip3 to install python-dateutil for salt"
+    #echo "Using pip3 to install python-dateutil for salt"
-    pip3 install -t /usr/lib/python3.6/site-packages/ python-dateutil
+    #pip3 install -t /usr/lib/python3.6/site-packages/ python-dateutil
     yum clean expire-cache
     yum -y install salt-minion-2019.2.2 yum-utils device-mapper-persistent-data lvm2 openssl
     yum -y update exclude=salt*
@@ -1010,8 +1011,8 @@ salt_install_mysql_deps() {
 
   if [ $OS == 'centos' ]; then
     yum -y install mariadb-devel
-    echo "Using pip3 to install mysqlclient for salt"
+    #echo "Using pip3 to install mysqlclient for salt"
-    pip3 install -t /usr/lib64/python3.6/site-packages/ mysqlclient
+    #pip3 install -t /usr/lib64/python3.6/site-packages/ mysqlclient
   elif [ $OS == 'ubuntu' ]; then
     apt-get -y install libmysqlclient-dev
     echo "Using pip3 to install mysqlclient for salt"