Merge branch 'dev' of https://github.com/Security-Onion-Solutions/securityonion-saltstack into feature/ubuntu18

pillar/logstash/eval.sls

@@ -25,3 +25,4 @@ logstash:
     - so/logstash-ossec-template.json
     - so/logstash-strelka-template.json
     - so/logstash-template.json
+    - so/logstash-bro-template.json

salt/common/tools/sbin/so-nodered-restart

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-restart nodered $1

salt/common/tools/sbin/so-nodered-start

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-start nodered $1
+

salt/common/tools/sbin/so-nodered-stop

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-stop nodered $1

salt/hive/thehive/etc/application.conf

@@ -209,7 +209,10 @@ misp {
   #} ## <-- Uncomment to complete the configuration
 }
 webhooks {
-  SOCtopusWebHook {
-    url = "http://{{ MASTERIP }}:7000/enrich"
+  NodeRedWebHook {
+    url = "http://{{ MASTERIP }}:1880/thehive"
   }
+  #SOCtopusWebHook {
+  #  url = "http://{{ MASTERIP }}:7000/enrich"
+  #}
 }

salt/logstash/pipelines/config/so/9000_output_bro.conf.jinja

@@ -23,8 +23,8 @@ output {
       pipeline => "%{event_type}"
       hosts => "{{ ES }}"
       index => "logstash-bro-%{+YYYY.MM.dd}"
-      template_name => "logstash"
-      template => "/logstash-template.json"
+      template_name => "logstash-bro"
+      template => "/logstash-bro-template.json"
       template_overwrite => true
     }
   }

salt/nodered/files/nodered_load_flows

@@ -0,0 +1,11 @@
+{%- set ip = salt['pillar.get']('static:masterip', '') -%}
+#!/bin/bash
+
+echo "Waiting for connection"
+until $(curl --output /dev/null --silent --head http://{{ ip }}:1880); do
+    echo '.'
+    sleep 1
+done
+echo "Loading flows..."
+curl -XPOST -v -H "Content-Type: application/json" -d @/opt/so/saltstack/salt/nodered/so_flows.json {{ ip }}:1880/flows
+echo "Done loading..."

salt/nodered/files/so_flows.json

@@ -0,0 +1,4 @@
+{%- set MASTERIP = salt['pillar.get']('static:masterip', '') -%}
+{%- set HIVEKEY = salt['pillar.get']('static:hivekey', '') -%}
+{%- set CORTEXKEY = salt['pillar.get']('static:cortexkey', '') -%}
+[{"id":"dca608c3.7d8af8","type":"tab","label":"TheHive - Webhook Events","disabled":false,"info":""},{"id":"4db74fa6.2556d","type":"tls-config","z":"","name":"","cert":"","key":"","ca":"","certname":"","keyname":"","caname":"","servername":"","verifyservercert":false},{"id":"aa6cf50d.a02fc8","type":"http in","z":"dca608c3.7d8af8","name":"TheHive Listener","url":"/thehive","method":"post","upload":false,"swaggerDoc":"","x":120,"y":780,"wires":[["2b92aebb.853dc2","2fce29bb.1b1376","82ad0f08.7a53f"]]},{"id":"2b92aebb.853dc2","type":"debug","z":"dca608c3.7d8af8","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","x":470,"y":940,"wires":[]},{"id":"a4ecb84a.805958","type":"switch","z":"dca608c3.7d8af8","name":"Operation","property":"payload.operation","propertyType":"msg","rules":[{"t":"eq","v":"Creation","vt":"str"},{"t":"eq","v":"Update","vt":"str"},{"t":"eq","v":"Delete","vt":"str"}],"checkall":"false","repair":false,"outputs":3,"x":580,"y":780,"wires":[["f1e954fd.3c21d8"],["65928861.c90a48"],["a259a26c.a21"]],"outputLabels":["Creation","Update","Delete"]},{"id":"f1e954fd.3c21d8","type":"switch","z":"dca608c3.7d8af8","name":"Creation","property":"payload.objectType","propertyType":"msg","rules":[{"t":"eq","v":"case","vt":"str"},{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"case_task","vt":"str"},{"t":"eq","v":"case_task_log","vt":"str"},{"t":"eq","v":"case_artifact_job","vt":"str"},{"t":"eq","v":"alert","vt":"str"},{"t":"eq","v":"user","vt":"str"}],"checkall":"false","repair":false,"outputs":7,"x":900,"y":480,"wires":[["e88b4cc2.f6afe"],["8c54e39.a1b4f2"],["64203fe8.e0ad5"],["3511de51.889a02"],["14544a8b.b6b2f5"],["44c595a4.45d45c"],["3eb4bedf.6e20a2"]],"inputLabels":["Operation"],"outputLabels":["case","case_artifact","case_task","case_task_log","action","alert","user"],"info":"No webhook data is received for the following events:\n\n- Creation of Dashboard\n- Creation of Case Templates\n"},{"id":"65928861.c90a48","type":"switch","z":"dca608c3.7d8af8","name":"Update","property":"payload.objectType","propertyType":"msg","rules":[{"t":"eq","v":"case","vt":"str"},{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"case_artifact_job","vt":"str"},{"t":"eq","v":"case_task","vt":"str"},{"t":"eq","v":"case_task_log","vt":"str"},{"t":"eq","v":"alert","vt":"str"},{"t":"eq","v":"user","vt":"str"}],"checkall":"false","repair":false,"outputs":7,"x":900,"y":860,"wires":[["eebe1748.1cd348"],["d703adc0.12fd1"],["2b738415.408d4c"],["6d97371a.406348"],["4ae621e1.9ae6"],["5786cee2.98109"],["54077728.447648"]],"inputLabels":["Operation"],"outputLabels":["case","case_artifact",null,"case_task","case_task_log","alert","user"]},{"id":"a259a26c.a21","type":"switch","z":"dca608c3.7d8af8","name":"Delete","property":"payload.objectType","propertyType":"msg","rules":[{"t":"eq","v":"case","vt":"str"},{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"case_task_log","vt":"str"}],"checkall":"false","repair":false,"outputs":3,"x":890,"y":1200,"wires":[["60c8bcfb.eff1f4"],["df708bab.348308"],["e9a8650c.e20cc8"]],"outputLabels":["case","case_artifact",""],"info":"Deleting a case task doesnt actually trigger a delete event.  It triggers an `update` event where the status = cancelled"},{"id":"54077728.447648","type":"switch","z":"dca608c3.7d8af8","name":"User","property":"payload.object.status","propertyType":"msg","rules":[{"t":"eq","v":"Locked","vt":"str"},{"t":"eq","v":"Ok","vt":"str"}],"checkall":"false","repair":false,"outputs":2,"x":1130,"y":980,"wires":[["9429d6c5.5ac788"],["4e3e091c.d35388"]]},{"id":"9429d6c5.5ac788","type":"function","z":"dca608c3.7d8af8","name":"status: Locked","func":"msg.topic = \"[The Hive] A user account was locked\";\nmsg.from = \"from@example.com\";\nmsg.to = \"to@example.com\";\nreturn msg;","outputs":1,"noerr":0,"x":1380,"y":972,"wires":[[]],"info":"- User account was locked"},{"id":"4e3e091c.d35388","type":"function","z":"dca608c3.7d8af8","name":"status: Ok","func":"msg.topic = \"[The Hive] A user account was changed\";\nmsg.from = \"from@example.com\";\nmsg.to = \"to@example.com\";\nreturn msg;","outputs":1,"noerr":0,"x":1360,"y":1020,"wires":[[]],"info":"- User account was unlocked\n- User description was changed\n- User role was changed\n- User API key was added\n- User API key was revoked\n"},{"id":"485f3be.1ffcfc4","type":"function","z":"dca608c3.7d8af8","name":"status: Open","func":"// Fires when a Case is updated AND status = open\n// This can include things like TLP/PAP changes\n\nreturn msg;","outputs":1,"noerr":0,"x":1370,"y":660,"wires":[[]]},{"id":"eebe1748.1cd348","type":"switch","z":"dca608c3.7d8af8","name":"case","property":"payload.object.status","propertyType":"msg","rules":[{"t":"eq","v":"Open","vt":"str"}],"checkall":"true","repair":false,"outputs":1,"x":1130,"y":740,"wires":[["485f3be.1ffcfc4","e4b7b4bf.2fb828"]],"info":"- A case was modified"},{"id":"8c54e39.a1b4f2","type":"switch","z":"dca608c3.7d8af8",Line truncated

salt/nodered/init.sls

@@ -34,6 +34,23 @@ nodered:
 #    - mode: 775
 #    - makedirs: True
 
+noderedflows:
+  file.recurse:
+    - name: /opt/so/saltstack/salt/nodered/
+    - source: salt://nodered/files
+    - user: 947
+    - group: 939
+    - template: jinja
+
+noderedflowsload:
+  file.managed:
+    - name: /usr/sbin/so-nodered-load-flows
+    - source: salt://nodered/files/nodered_load_flows
+    - user: 0
+    - group: 0
+    - mode: 755
+    - template: jinja
+
 noderedlog:
   file.directory:
     - name: /opt/so/log/nodered
@@ -44,10 +61,15 @@ noderedlog:
 
 so-nodered:
   docker_container.running:
-    - image: soshybridhunter/so-nodered:HH1.1.5
+    - image: soshybridhunter/so-nodered:HH1.2.1
     - interactive: True
     - binds:
       - /opt/so/conf/nodered/:/data:rw
     - port_bindings:
       - 0.0.0.0:1880:1880
 
+so-nodered-flows:
+  cmd.run:
+    - name: /usr/sbin/so-nodered-load-flows
+    - cwd: /
+

salt/playbook/init.sls

@@ -52,3 +52,10 @@ so-navigator:
     - identifier: so-playbook-sync
     - user: root
     - minute: '*/5'
+
+/usr/sbin/so-playbook-ruleupdate:
+  cron.present:
+    - identifier: so-playbook-ruleupdate
+    - user: root
+    - minute: '1'
+    - hour: '6'

salt/zeek/init.sls

@@ -48,15 +48,9 @@ zeekextractdir:
     - group: 939
     - makedirs: True
 
-zeeksfafincompletedir:
+zeekextractcompletedir:
   file.directory:
-    - name: /nsm/faf/files/incomplete
-    - user: 937
-    - makedirs: true
-
-zeeksfafcompletedir:
-  file.directory:
-    - name: /nsm/faf/files/complete
+    - name: /nsm/zeek/extracted/complete
     - user: 937
     - makedirs: true
 

salt/zeek/policy/securityonion/file-extraction/extract.zeek

@@ -1,21 +1,80 @@
-global ext_map: table[string] of string = {
-    ["application/x-dosexec"] = "exe",
-    ["text/plain"] = "txt",
-    ["image/jpeg"] = "jpg",
-    ["image/png"] = "png",
-    ["text/html"] = "html",
-} &default ="";
-
-event file_sniff(f: fa_file, meta: fa_metadata)
-    {
-    if ( ! meta?$mime_type || meta$mime_type != "application/x-dosexec" )
-        return;
-
+# Directory to stage Zeek extracted files before processing
+redef FileExtract::prefix = "/nsm/zeek/extracted/";
+# Set a limit to the file size
+redef FileExtract::default_limit = 9000000;
+# These are the mimetypes we want to rip off the networks
+export {
+    global _mime_whitelist: table[string] of string = {
+        ["application/x-dosexec"] = "exe",
+        ["application/pdf"] = "pdf",
+        ["application/msword"] = "doc",
+        ["application/vnd.ms-powerpoint"] = "doc",
+        ["application/rtf"] = "doc",
+        ["application/vnd.ms-word.document.macroenabled.12"] = "doc",
+        ["application/vnd.ms-word.template.macroenabled.12"] = "doc",
+        ["application/vnd.ms-powerpoint.template.macroenabled.12"] = "doc",
+        ["application/vnd.ms-excel"] = "doc",
+        ["application/vnd.ms-excel.addin.macroenabled.12"] = "doc",
+        ["application/vnd.ms-excel.sheet.binary.macroenabled.12"] = "doc",
+        ["application/vnd.ms-excel.template.macroenabled.12"] = "doc",
+        ["application/vnd.ms-excel.sheet.macroenabled.12"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.presentationml.presentation"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.presentationml.slide"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.presentationml.slideshow"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.presentationml.template"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.spreadsheetml.template"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.wordprocessingml.template"] = "doc",
+        ["application/vnd.ms-powerpoint.addin.macroenabled.12"] = "doc",
+        ["application/vnd.ms-powerpoint.slide.macroenabled.12"] = "doc",
+        ["application/vnd.ms-powerpoint.presentation.macroenabled.12"] = "doc",
+        ["application/vnd.ms-powerpoint.slideshow.macroenabled.12"] = "doc",
+        ["application/vnd.openxmlformats-officedocument"] = "doc"
+        # Need to add other types such as zip, ps1, etc
+        };
+}
+# Start grabbing the file from the network if it matches the mimetype
+event file_sniff(f: fa_file, meta: fa_metadata) &priority=10 {
     local ext = "";
-
-    if ( meta?$mime_type )
-        ext = ext_map[meta$mime_type];
-
-    local fname = fmt("/nsm/zeek/extracted/%s-%s.%s", f$source, f$id, ext);
-    Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
+    if( meta?$mime_type ) {
+    if ( meta$mime_type !in _mime_whitelist ) {
+          return;
     }
+    ext = _mime_whitelist[meta$mime_type];
+    local fname = fmt("%s-%s.%s", f$source, f$id, ext);
+    Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
+        }
+}
+# Wait for file_state_remove before you do anything. This is when it is actually done.
+event file_state_remove(f: fa_file)
+        {
+        if ( !f$info?$extracted || FileExtract::prefix == "" ) {
+                return;
+        }
+        # Check some conditions so we know the file is intact:
+        # Check for MD5
+        # Check for total_bytes
+        # Check for missing bytes
+        # Check if timed out
+        if ( !f$info?$md5 || !f?$total_bytes || f$missing_bytes > 0 || f$info$timedout) {
+          # Delete the file if it didn't pass our requirements check.
+
+          local nuke = fmt("rm %s/%s", FileExtract::prefix, f$info$extracted);
+          when ( local nukeit = Exec::run([$cmd=nuke]) )
+                    {
+                    }
+                    return;
+        }
+        local orig = f$info$extracted;
+        local split_orig = split_string(f$info$extracted, /\./);
+        local extension = split_orig[|split_orig|-1];
+        local dest = fmt("%scomplete/%s-%s-%s.%s", FileExtract::prefix, f$source, f$id, f$info$md5, extension);
+        # Copy it to the $prefix/complete folder then delete it. I got some weird results with moving when it came to watchdog in python.
+        local cmd = fmt("cp %s/%s %s && rm %s/%s", FileExtract::prefix, orig, dest, FileExtract::prefix, orig);
+      when ( local result = Exec::run([$cmd=cmd]) )
+                {
+                }
+      f$info$extracted = dest;
+        }
+