Merge branch 'dev' of https://github.com/Security-Onion-Solutions/securityonion-saltstack into feature/ubuntu18

pillar/logstash/eval.sls

@@ -25,3 +25,4 @@ logstash:
     - so/logstash-ossec-template.json
     - so/logstash-strelka-template.json
     - so/logstash-template.json
+    - so/logstash-bro-template.json

salt/common/tools/sbin/so-nodered-restart

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-restart nodered $1

salt/common/tools/sbin/so-nodered-start

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-start nodered $1
+

salt/common/tools/sbin/so-nodered-stop

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-stop nodered $1

salt/hive/thehive/etc/application.conf

@@ -209,7 +209,10 @@ misp {
   #} ## <-- Uncomment to complete the configuration
 }
 webhooks {
-  SOCtopusWebHook {
+  NodeRedWebHook {
-    url = "http://{{ MASTERIP }}:7000/enrich"
+    url = "http://{{ MASTERIP }}:1880/thehive"
   }
+  #SOCtopusWebHook {
+  #  url = "http://{{ MASTERIP }}:7000/enrich"
+  #}
 }

salt/logstash/pipelines/config/so/9000_output_bro.conf.jinja

@@ -23,8 +23,8 @@ output {
       pipeline => "%{event_type}"
       hosts => "{{ ES }}"
       index => "logstash-bro-%{+YYYY.MM.dd}"
-      template_name => "logstash"
+      template_name => "logstash-bro"
-      template => "/logstash-template.json"
+      template => "/logstash-bro-template.json"
       template_overwrite => true
     }
   }

salt/nodered/files/nodered_load_flows

@@ -0,0 +1,11 @@
+{%- set ip = salt['pillar.get']('static:masterip', '') -%}
+#!/bin/bash
+
+echo "Waiting for connection"
+until $(curl --output /dev/null --silent --head http://{{ ip }}:1880); do
+    echo '.'
+    sleep 1
+done
+echo "Loading flows..."
+curl -XPOST -v -H "Content-Type: application/json" -d @/opt/so/saltstack/salt/nodered/so_flows.json {{ ip }}:1880/flows
+echo "Done loading..."

salt/nodered/init.sls

@@ -34,6 +34,23 @@ nodered:
 #    - mode: 775
 #    - makedirs: True
 
+noderedflows:
+  file.recurse:
+    - name: /opt/so/saltstack/salt/nodered/
+    - source: salt://nodered/files
+    - user: 947
+    - group: 939
+    - template: jinja
+
+noderedflowsload:
+  file.managed:
+    - name: /usr/sbin/so-nodered-load-flows
+    - source: salt://nodered/files/nodered_load_flows
+    - user: 0
+    - group: 0
+    - mode: 755
+    - template: jinja
+
 noderedlog:
   file.directory:
     - name: /opt/so/log/nodered
@@ -44,10 +61,15 @@ noderedlog:
 
 so-nodered:
   docker_container.running:
-    - image: soshybridhunter/so-nodered:HH1.1.5
+    - image: soshybridhunter/so-nodered:HH1.2.1
     - interactive: True
     - binds:
       - /opt/so/conf/nodered/:/data:rw
     - port_bindings:
       - 0.0.0.0:1880:1880
 
+so-nodered-flows:
+  cmd.run:
+    - name: /usr/sbin/so-nodered-load-flows
+    - cwd: /
+

salt/playbook/init.sls

@@ -52,3 +52,10 @@ so-navigator:
     - identifier: so-playbook-sync
     - user: root
     - minute: '*/5'
+
+/usr/sbin/so-playbook-ruleupdate:
+  cron.present:
+    - identifier: so-playbook-ruleupdate
+    - user: root
+    - minute: '1'
+    - hour: '6'

salt/zeek/init.sls

@@ -48,15 +48,9 @@ zeekextractdir:
     - group: 939
     - makedirs: True
 
-zeeksfafincompletedir:
+zeekextractcompletedir:
   file.directory:
-    - name: /nsm/faf/files/incomplete
+    - name: /nsm/zeek/extracted/complete
-    - user: 937
-    - makedirs: true
-
-zeeksfafcompletedir:
-  file.directory:
-    - name: /nsm/faf/files/complete
     - user: 937
     - makedirs: true
 

salt/zeek/policy/securityonion/file-extraction/extract.zeek

@@ -1,21 +1,80 @@
-global ext_map: table[string] of string = {
+# Directory to stage Zeek extracted files before processing
-    ["application/x-dosexec"] = "exe",
+redef FileExtract::prefix = "/nsm/zeek/extracted/";
-    ["text/plain"] = "txt",
+# Set a limit to the file size
-    ["image/jpeg"] = "jpg",
+redef FileExtract::default_limit = 9000000;
-    ["image/png"] = "png",
+# These are the mimetypes we want to rip off the networks
-    ["text/html"] = "html",
+export {
-} &default ="";
+    global _mime_whitelist: table[string] of string = {
-
+        ["application/x-dosexec"] = "exe",
-event file_sniff(f: fa_file, meta: fa_metadata)
+        ["application/pdf"] = "pdf",
-    {
+        ["application/msword"] = "doc",
-    if ( ! meta?$mime_type || meta$mime_type != "application/x-dosexec" )
+        ["application/vnd.ms-powerpoint"] = "doc",
-        return;
+        ["application/rtf"] = "doc",
-
+        ["application/vnd.ms-word.document.macroenabled.12"] = "doc",
+        ["application/vnd.ms-word.template.macroenabled.12"] = "doc",
+        ["application/vnd.ms-powerpoint.template.macroenabled.12"] = "doc",
+        ["application/vnd.ms-excel"] = "doc",
+        ["application/vnd.ms-excel.addin.macroenabled.12"] = "doc",
+        ["application/vnd.ms-excel.sheet.binary.macroenabled.12"] = "doc",
+        ["application/vnd.ms-excel.template.macroenabled.12"] = "doc",
+        ["application/vnd.ms-excel.sheet.macroenabled.12"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.presentationml.presentation"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.presentationml.slide"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.presentationml.slideshow"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.presentationml.template"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.spreadsheetml.template"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"] = "doc",
+        ["application/vnd.openxmlformats-officedocument.wordprocessingml.template"] = "doc",
+        ["application/vnd.ms-powerpoint.addin.macroenabled.12"] = "doc",
+        ["application/vnd.ms-powerpoint.slide.macroenabled.12"] = "doc",
+        ["application/vnd.ms-powerpoint.presentation.macroenabled.12"] = "doc",
+        ["application/vnd.ms-powerpoint.slideshow.macroenabled.12"] = "doc",
+        ["application/vnd.openxmlformats-officedocument"] = "doc"
+        # Need to add other types such as zip, ps1, etc
+        };
+}
+# Start grabbing the file from the network if it matches the mimetype
+event file_sniff(f: fa_file, meta: fa_metadata) &priority=10 {
     local ext = "";
-
+    if( meta?$mime_type ) {
-    if ( meta?$mime_type )
+    if ( meta$mime_type !in _mime_whitelist ) {
-        ext = ext_map[meta$mime_type];
+          return;
-
-    local fname = fmt("/nsm/zeek/extracted/%s-%s.%s", f$source, f$id, ext);
-    Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
     }
+    ext = _mime_whitelist[meta$mime_type];
+    local fname = fmt("%s-%s.%s", f$source, f$id, ext);
+    Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
+        }
+}
+# Wait for file_state_remove before you do anything. This is when it is actually done.
+event file_state_remove(f: fa_file)
+        {
+        if ( !f$info?$extracted || FileExtract::prefix == "" ) {
+                return;
+        }
+        # Check some conditions so we know the file is intact:
+        # Check for MD5
+        # Check for total_bytes
+        # Check for missing bytes
+        # Check if timed out
+        if ( !f$info?$md5 || !f?$total_bytes || f$missing_bytes > 0 || f$info$timedout) {
+          # Delete the file if it didn't pass our requirements check.
+
+          local nuke = fmt("rm %s/%s", FileExtract::prefix, f$info$extracted);
+          when ( local nukeit = Exec::run([$cmd=nuke]) )
+                    {
+                    }
+                    return;
+        }
+        local orig = f$info$extracted;
+        local split_orig = split_string(f$info$extracted, /\./);
+        local extension = split_orig[|split_orig|-1];
+        local dest = fmt("%scomplete/%s-%s-%s.%s", FileExtract::prefix, f$source, f$id, f$info$md5, extension);
+        # Copy it to the $prefix/complete folder then delete it. I got some weird results with moving when it came to watchdog in python.
+        local cmd = fmt("cp %s/%s %s && rm %s/%s", FileExtract::prefix, orig, dest, FileExtract::prefix, orig);
+      when ( local result = Exec::run([$cmd=cmd]) )
+                {
+                }
+      f$info$extracted = dest;
+        }
+