diff --git a/pillar/logstash/eval.sls b/pillar/logstash/eval.sls
index 7f817ed39..f6cf222b3 100644
--- a/pillar/logstash/eval.sls
+++ b/pillar/logstash/eval.sls
@@ -25,3 +25,4 @@ logstash:
- so/logstash-ossec-template.json
- so/logstash-strelka-template.json
- so/logstash-template.json
+ - so/logstash-bro-template.json
diff --git a/salt/common/tools/sbin/so-auth-restart b/salt/common/tools/sbin/so-auth-restart
old mode 100644
new mode 100755
diff --git a/salt/common/tools/sbin/so-auth-start b/salt/common/tools/sbin/so-auth-start
old mode 100644
new mode 100755
diff --git a/salt/common/tools/sbin/so-auth-stop b/salt/common/tools/sbin/so-auth-stop
old mode 100644
new mode 100755
diff --git a/salt/common/tools/sbin/so-nodered-restart b/salt/common/tools/sbin/so-nodered-restart
new file mode 100755
index 000000000..b30e7d4bb
--- /dev/null
+++ b/salt/common/tools/sbin/so-nodered-restart
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-restart nodered $1
diff --git a/salt/common/tools/sbin/so-nodered-start b/salt/common/tools/sbin/so-nodered-start
new file mode 100755
index 000000000..5d21022cf
--- /dev/null
+++ b/salt/common/tools/sbin/so-nodered-start
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-start nodered $1
+
diff --git a/salt/common/tools/sbin/so-nodered-stop b/salt/common/tools/sbin/so-nodered-stop
new file mode 100755
index 000000000..9a83d6cef
--- /dev/null
+++ b/salt/common/tools/sbin/so-nodered-stop
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-stop nodered $1
diff --git a/salt/hive/thehive/etc/application.conf b/salt/hive/thehive/etc/application.conf
index 14a635e54..ae588a42d 100644
--- a/salt/hive/thehive/etc/application.conf
+++ b/salt/hive/thehive/etc/application.conf
@@ -209,7 +209,10 @@ misp {
#} ## <-- Uncomment to complete the configuration
}
webhooks {
- SOCtopusWebHook {
- url = "http://{{ MASTERIP }}:7000/enrich"
+ NodeRedWebHook {
+ url = "http://{{ MASTERIP }}:1880/thehive"
}
+ #SOCtopusWebHook {
+ # url = "http://{{ MASTERIP }}:7000/enrich"
+ #}
}
diff --git a/salt/logstash/pipelines/config/so/9000_output_bro.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_bro.conf.jinja
index 553500281..acc31ae00 100644
--- a/salt/logstash/pipelines/config/so/9000_output_bro.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9000_output_bro.conf.jinja
@@ -23,8 +23,8 @@ output {
pipeline => "%{event_type}"
hosts => "{{ ES }}"
index => "logstash-bro-%{+YYYY.MM.dd}"
- template_name => "logstash"
- template => "/logstash-template.json"
+ template_name => "logstash-bro"
+ template => "/logstash-bro-template.json"
template_overwrite => true
}
}
diff --git a/salt/nodered/files/nodered_load_flows b/salt/nodered/files/nodered_load_flows
new file mode 100644
index 000000000..c48fcd692
--- /dev/null
+++ b/salt/nodered/files/nodered_load_flows
@@ -0,0 +1,11 @@
+{%- set ip = salt['pillar.get']('static:masterip', '') -%}
+#!/bin/bash
+
+echo "Waiting for connection"
+until $(curl --output /dev/null --silent --head http://{{ ip }}:1880); do
+ echo '.'
+ sleep 1
+done
+echo "Loading flows..."
+curl -XPOST -v -H "Content-Type: application/json" -d @/opt/so/saltstack/salt/nodered/so_flows.json {{ ip }}:1880/flows
+echo "Done loading..."
diff --git a/salt/nodered/files/so_flows.json b/salt/nodered/files/so_flows.json
new file mode 100644
index 000000000..8ab8cbf81
--- /dev/null
+++ b/salt/nodered/files/so_flows.json
@@ -0,0 +1,4 @@
+{%- set MASTERIP = salt['pillar.get']('static:masterip', '') -%}
+{%- set HIVEKEY = salt['pillar.get']('static:hivekey', '') -%}
+{%- set CORTEXKEY = salt['pillar.get']('static:cortexkey', '') -%}
+[{"id":"dca608c3.7d8af8","type":"tab","label":"TheHive - Webhook Events","disabled":false,"info":""},{"id":"4db74fa6.2556d","type":"tls-config","z":"","name":"","cert":"","key":"","ca":"","certname":"","keyname":"","caname":"","servername":"","verifyservercert":false},{"id":"aa6cf50d.a02fc8","type":"http in","z":"dca608c3.7d8af8","name":"TheHive Listener","url":"/thehive","method":"post","upload":false,"swaggerDoc":"","x":120,"y":780,"wires":[["2b92aebb.853dc2","2fce29bb.1b1376","82ad0f08.7a53f"]]},{"id":"2b92aebb.853dc2","type":"debug","z":"dca608c3.7d8af8","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","x":470,"y":940,"wires":[]},{"id":"a4ecb84a.805958","type":"switch","z":"dca608c3.7d8af8","name":"Operation","property":"payload.operation","propertyType":"msg","rules":[{"t":"eq","v":"Creation","vt":"str"},{"t":"eq","v":"Update","vt":"str"},{"t":"eq","v":"Delete","vt":"str"}],"checkall":"false","repair":false,"outputs":3,"x":580,"y":780,"wires":[["f1e954fd.3c21d8"],["65928861.c90a48"],["a259a26c.a21"]],"outputLabels":["Creation","Update","Delete"]},{"id":"f1e954fd.3c21d8","type":"switch","z":"dca608c3.7d8af8","name":"Creation","property":"payload.objectType","propertyType":"msg","rules":[{"t":"eq","v":"case","vt":"str"},{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"case_task","vt":"str"},{"t":"eq","v":"case_task_log","vt":"str"},{"t":"eq","v":"case_artifact_job","vt":"str"},{"t":"eq","v":"alert","vt":"str"},{"t":"eq","v":"user","vt":"str"}],"checkall":"false","repair":false,"outputs":7,"x":900,"y":480,"wires":[["e88b4cc2.f6afe"],["8c54e39.a1b4f2"],["64203fe8.e0ad5"],["3511de51.889a02"],["14544a8b.b6b2f5"],["44c595a4.45d45c"],["3eb4bedf.6e20a2"]],"inputLabels":["Operation"],"outputLabels":["case","case_artifact","case_task","case_task_log","action","alert","user"],"info":"No webhook data is received for the following events:\n\n- Creation of Dashboard\n- Creation of Case Templates\n"},{"id":"65928861.c90a48","type":"switch","z":"dca608c3.7d8af8","name":"Update","property":"payload.objectType","propertyType":"msg","rules":[{"t":"eq","v":"case","vt":"str"},{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"case_artifact_job","vt":"str"},{"t":"eq","v":"case_task","vt":"str"},{"t":"eq","v":"case_task_log","vt":"str"},{"t":"eq","v":"alert","vt":"str"},{"t":"eq","v":"user","vt":"str"}],"checkall":"false","repair":false,"outputs":7,"x":900,"y":860,"wires":[["eebe1748.1cd348"],["d703adc0.12fd1"],["2b738415.408d4c"],["6d97371a.406348"],["4ae621e1.9ae6"],["5786cee2.98109"],["54077728.447648"]],"inputLabels":["Operation"],"outputLabels":["case","case_artifact",null,"case_task","case_task_log","alert","user"]},{"id":"a259a26c.a21","type":"switch","z":"dca608c3.7d8af8","name":"Delete","property":"payload.objectType","propertyType":"msg","rules":[{"t":"eq","v":"case","vt":"str"},{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"case_task_log","vt":"str"}],"checkall":"false","repair":false,"outputs":3,"x":890,"y":1200,"wires":[["60c8bcfb.eff1f4"],["df708bab.348308"],["e9a8650c.e20cc8"]],"outputLabels":["case","case_artifact",""],"info":"Deleting a case task doesnt actually trigger a delete event. It triggers an `update` event where the status = cancelled"},{"id":"54077728.447648","type":"switch","z":"dca608c3.7d8af8","name":"User","property":"payload.object.status","propertyType":"msg","rules":[{"t":"eq","v":"Locked","vt":"str"},{"t":"eq","v":"Ok","vt":"str"}],"checkall":"false","repair":false,"outputs":2,"x":1130,"y":980,"wires":[["9429d6c5.5ac788"],["4e3e091c.d35388"]]},{"id":"9429d6c5.5ac788","type":"function","z":"dca608c3.7d8af8","name":"status: Locked","func":"msg.topic = \"[The Hive] A user account was locked\";\nmsg.from = \"from@example.com\";\nmsg.to = \"to@example.com\";\nreturn msg;","outputs":1,"noerr":0,"x":1380,"y":972,"wires":[[]],"info":"- User account was locked"},{"id":"4e3e091c.d35388","type":"function","z":"dca608c3.7d8af8","name":"status: Ok","func":"msg.topic = \"[The Hive] A user account was changed\";\nmsg.from = \"from@example.com\";\nmsg.to = \"to@example.com\";\nreturn msg;","outputs":1,"noerr":0,"x":1360,"y":1020,"wires":[[]],"info":"- User account was unlocked\n- User description was changed\n- User role was changed\n- User API key was added\n- User API key was revoked\n"},{"id":"485f3be.1ffcfc4","type":"function","z":"dca608c3.7d8af8","name":"status: Open","func":"// Fires when a Case is updated AND status = open\n// This can include things like TLP/PAP changes\n\nreturn msg;","outputs":1,"noerr":0,"x":1370,"y":660,"wires":[[]]},{"id":"eebe1748.1cd348","type":"switch","z":"dca608c3.7d8af8","name":"case","property":"payload.object.status","propertyType":"msg","rules":[{"t":"eq","v":"Open","vt":"str"}],"checkall":"true","repair":false,"outputs":1,"x":1130,"y":740,"wires":[["485f3be.1ffcfc4","e4b7b4bf.2fb828"]],"info":"- A case was modified"},{"id":"8c54e39.a1b4f2","type":"switch","z":"dca608c3.7d8af8","name":"case_artifact: Run Analyzer","property":"payload.object.dataType","propertyType":"msg","rules":[{"t":"eq","v":"ip","vt":"str"},{"t":"eq","v":"domain","vt":"str"}],"checkall":"true","repair":false,"outputs":2,"x":1600,"y":340,"wires":[["eb8cfeb7.a7118","a5dd8a8a.065b88"],["eb8cfeb7.a7118","a5dd8a8a.065b88"]],"info":"# References\n\n\n"},{"id":"2fce29bb.1b1376","type":"function","z":"dca608c3.7d8af8","name":"Add headers","func":"msg.thehive_url = 'https://{{ MASTERIP }}/thehive';\nmsg.cortex_url = 'https://{{ MASTERIP }}/cortex';\nmsg.cortex_id = 'CORTEX-SERVER-ID';\nreturn msg;","outputs":1,"noerr":0,"x":350,"y":780,"wires":[["a4ecb84a.805958"]]},{"id":"e4b7b4bf.2fb828","type":"function","z":"dca608c3.7d8af8","name":"status: Resolved","func":"// Fires when a case is closed (resolved)\n\nreturn msg;","outputs":1,"noerr":0,"x":1390,"y":720,"wires":[[]]},{"id":"e88b4cc2.f6afe","type":"function","z":"dca608c3.7d8af8","name":"case","func":"// Fires when a case is created\n// or when a responder is generated against a case\n\nreturn msg;","outputs":1,"noerr":0,"x":1130,"y":320,"wires":[[]]},{"id":"64203fe8.e0ad5","type":"function","z":"dca608c3.7d8af8","name":"case_task","func":"// Fires when a case task is created\nreturn msg;","outputs":1,"noerr":0,"x":1140,"y":400,"wires":[[]]},{"id":"3511de51.889a02","type":"function","z":"dca608c3.7d8af8","name":"case_task_log","func":"// Fires when a case task log is created\n\nreturn msg;","outputs":1,"noerr":0,"x":1163,"y":440,"wires":[[]]},{"id":"14544a8b.b6b2f5","type":"function","z":"dca608c3.7d8af8","name":"case_artifact_job","func":"// Fires when a Responder or Analyzser is Run on an existing observable\n\nreturn msg;","outputs":1,"noerr":0,"x":1173,"y":480,"wires":[[]]},{"id":"2b738415.408d4c","type":"function","z":"dca608c3.7d8af8","name":"case_artifact_job","func":"\nreturn msg;","outputs":1,"noerr":0,"x":1170,"y":820,"wires":[[]]},{"id":"3eb4bedf.6e20a2","type":"function","z":"dca608c3.7d8af8","name":"user","func":"// Fires when a user is created\n\nreturn msg;","outputs":1,"noerr":0,"x":1133,"y":560,"wires":[[]]},{"id":"d703adc0.12fd1","type":"function","z":"dca608c3.7d8af8","name":"case_artifact","func":"// Fires when an artifact is updated\nreturn msg;","outputs":1,"noerr":0,"x":1150,"y":780,"wires":[[]]},{"id":"6d97371a.406348","type":"function","z":"dca608c3.7d8af8","name":"case_task","func":"// Fires when a case task is updated\nreturn msg;","outputs":1,"noerr":0,"x":1140,"y":860,"wires":[[]]},{"id":"4ae621e1.9ae6","type":"function","z":"dca608c3.7d8af8","name":"case_task_log","func":"//Fires when a case_task_log is updated\n\nreturn msg;","outputs":1,"noerr":0,"x":1160,"y":900,"wires":[[]]},{"id":"60c8bcfb.eff1f4","type":"function","z":"dca608c3.7d8af8","name":"case","func":"//Fires when a case is deleted\nreturn msg;","outputs":1,"noerr":0,"x":1130,"y":1160,"wires":[[]]},{"id":"df708bab.348308","type":"function","z":"dca608c3.7d8af8","name":"case_artifact","func":"//Fires when a case_artifact is deleted\nreturn msg;","outputs":1,"noerr":0,"x":1150,"y":1200,"wires":[[]]},{"id":"e9a8650c.e20cc8","type":"function","z":"dca608c3.7d8af8","name":"case_task_log","func":"//Fires when a case_task_log is deleted\nreturn msg;","outputs":1,"noerr":0,"x":1160,"y":1240,"wires":[[]]},{"id":"5786cee2.98109","type":"function","z":"dca608c3.7d8af8","name":"alert","func":"//Fires when an alert is updated\nreturn msg;","outputs":1,"noerr":0,"x":1130,"y":940,"wires":[[]]},{"id":"44c595a4.45d45c","type":"change","z":"dca608c3.7d8af8","d":true,"name":"Convert Alert Msg to Artifacts","rules":[{"t":"move","p":"payload.object.artifacts","pt":"msg","to":"payload","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":1200,"y":520,"wires":[["6dcca25e.04bd2c"]]},{"id":"6dcca25e.04bd2c","type":"split","z":"dca608c3.7d8af8","name":"Split Artifacts","splt":"\\n","spltType":"str","arraySplt":1,"arraySpltType":"len","stream":false,"addname":"","x":1430,"y":520,"wires":[["767c84f2.c9ba2c"]]},{"id":"767c84f2.c9ba2c","type":"switch","z":"dca608c3.7d8af8","name":"alert: Run Analyzer","property":"payload.dataType","propertyType":"msg","rules":[{"t":"eq","v":"ip","vt":"str"},{"t":"eq","v":"domain","vt":"str"}],"checkall":"true","repair":false,"outputs":2,"x":1630,"y":400,"wires":[["eb8cfeb7.a7118","a5dd8a8a.065b88"],["a5dd8a8a.065b88","eb8cfeb7.a7118"]],"info":"# References\n\n\n"},{"id":"82ad0f08.7a53f","type":"http response","z":"dca608c3.7d8af8","name":"Ack Event Receipt","statusCode":"200","headers":{},"x":250,"y":940,"wires":[]},{"id":"a5dd8a8a.065b88","type":"function","z":"dca608c3.7d8af8","name":"Run Analyzer: CERT DNS","func":"msg.analyzer_id = \"4f28afc20d78f98df425e36e561af33f\";\n\nif (msg.payload.objectId) {\n msg.tag = \"case_artifact\"\n msg.artifact_id = msg.payload.objectId\n msg.url = msg.thehive_url + '/api/connector/cortex/job';\n msg.payload = {\n 'cortexId' : msg.cortex_id,\n 'artifactId': msg.artifact_id,\n 'analyzerId': msg.analyzer_id\n };\n}\nelse {\n msg.tag = \"observable\"\n msg.observable = msg.payload.data\n msg.dataType = msg.payload.dataType\n\n msg.url = msg.cortex_url + '/api/analyzer/' + msg.analyzer_id + '/run';\n msg.payload = {\n 'data' : msg.observable,\n 'dataType': msg.dataType \n };\n}\nreturn msg;","outputs":1,"noerr":0,"x":1930,"y":420,"wires":[["f050a09f.b2201"]]},{"id":"eb8cfeb7.a7118","type":"function","z":"dca608c3.7d8af8","name":"Run Analyzer: Urlscan","func":"msg.analyzer_id = \"54e51b62c6c8ddc3cbc3cbdd889a0557\";\n\nif (msg.payload.objectId) {\n msg.tag = \"case_artifact\"\n msg.artifact_id = msg.payload.objectId\n msg.url = msg.thehive_url + '/api/connector/cortex/job';\n msg.payload = {\n 'cortexId' : msg.cortex_id,\n 'artifactId': msg.artifact_id,\n 'analyzerId': msg.analyzer_id\n };\n}\nelse {\n msg.tag = \"observable\"\n msg.observable = msg.payload.data\n msg.dataType = msg.payload.dataType\n\n msg.url = msg.cortex_url + '/api/analyzer/' + msg.analyzer_id + '/run';\n msg.payload = {\n 'data' : msg.observable,\n 'dataType': msg.dataType \n };\n}\nreturn msg;","outputs":1,"noerr":0,"x":1920,"y":320,"wires":[["f050a09f.b2201"]]},{"id":"1c448528.3032fb","type":"http request","z":"dca608c3.7d8af8","name":"Submit to Cortex","method":"POST","ret":"obj","paytoqs":false,"url":"","tls":"4db74fa6.2556d","persist":false,"proxy":"","authType":"bearer","credentials": {"user": "", "password": "{{ CORTEXKEY }}"},"x":2450,"y":420,"wires":[["ea6614fb.752a78"]]},{"id":"ea6614fb.752a78","type":"debug","z":"dca608c3.7d8af8","name":"Debug","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","x":2670,"y":360,"wires":[]},{"id":"f050a09f.b2201","type":"switch","z":"dca608c3.7d8af8","name":"Cases vs Alerts","property":"tag","propertyType":"msg","rules":[{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"observable","vt":"str"}],"checkall":"true","repair":false,"outputs":2,"x":2200,"y":360,"wires":[["f7fca977.a73b28"],["1c448528.3032fb"]],"inputLabels":["Data"],"outputLabels":["Cases","Alerts"]},{"id":"f7fca977.a73b28","type":"http request","z":"dca608c3.7d8af8","name":"Submit to TheHive","method":"POST","ret":"obj","paytoqs":false,"url":"","tls":"4db74fa6.2556d","persist":false,"proxy":"","authType":"bearer","credentials": {"user": "", "password": "{{ HIVEKEY }}"},"x":2450,"y":280,"wires":[["ea6614fb.752a78"]]}]
diff --git a/salt/nodered/init.sls b/salt/nodered/init.sls
index 96aee2630..af58ab866 100644
--- a/salt/nodered/init.sls
+++ b/salt/nodered/init.sls
@@ -34,6 +34,23 @@ nodered:
# - mode: 775
# - makedirs: True
+noderedflows:
+ file.recurse:
+ - name: /opt/so/saltstack/salt/nodered/
+ - source: salt://nodered/files
+ - user: 947
+ - group: 939
+ - template: jinja
+
+noderedflowsload:
+ file.managed:
+ - name: /usr/sbin/so-nodered-load-flows
+ - source: salt://nodered/files/nodered_load_flows
+ - user: 0
+ - group: 0
+ - mode: 755
+ - template: jinja
+
noderedlog:
file.directory:
- name: /opt/so/log/nodered
@@ -44,10 +61,15 @@ noderedlog:
so-nodered:
docker_container.running:
- - image: soshybridhunter/so-nodered:HH1.1.5
+ - image: soshybridhunter/so-nodered:HH1.2.1
- interactive: True
- binds:
- /opt/so/conf/nodered/:/data:rw
- port_bindings:
- 0.0.0.0:1880:1880
+so-nodered-flows:
+ cmd.run:
+ - name: /usr/sbin/so-nodered-load-flows
+ - cwd: /
+
diff --git a/salt/playbook/init.sls b/salt/playbook/init.sls
index 049c5fab9..04c321ab7 100644
--- a/salt/playbook/init.sls
+++ b/salt/playbook/init.sls
@@ -52,3 +52,10 @@ so-navigator:
- identifier: so-playbook-sync
- user: root
- minute: '*/5'
+
+/usr/sbin/so-playbook-ruleupdate:
+ cron.present:
+ - identifier: so-playbook-ruleupdate
+ - user: root
+ - minute: '1'
+ - hour: '6'
diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls
index db54bf62e..88a1ecbde 100644
--- a/salt/zeek/init.sls
+++ b/salt/zeek/init.sls
@@ -48,15 +48,9 @@ zeekextractdir:
- group: 939
- makedirs: True
-zeeksfafincompletedir:
+zeekextractcompletedir:
file.directory:
- - name: /nsm/faf/files/incomplete
- - user: 937
- - makedirs: true
-
-zeeksfafcompletedir:
- file.directory:
- - name: /nsm/faf/files/complete
+ - name: /nsm/zeek/extracted/complete
- user: 937
- makedirs: true
diff --git a/salt/zeek/policy/securityonion/file-extraction/extract.zeek b/salt/zeek/policy/securityonion/file-extraction/extract.zeek
index b8e8478bd..fbb635982 100644
--- a/salt/zeek/policy/securityonion/file-extraction/extract.zeek
+++ b/salt/zeek/policy/securityonion/file-extraction/extract.zeek
@@ -1,21 +1,80 @@
-global ext_map: table[string] of string = {
- ["application/x-dosexec"] = "exe",
- ["text/plain"] = "txt",
- ["image/jpeg"] = "jpg",
- ["image/png"] = "png",
- ["text/html"] = "html",
-} &default ="";
-
-event file_sniff(f: fa_file, meta: fa_metadata)
- {
- if ( ! meta?$mime_type || meta$mime_type != "application/x-dosexec" )
- return;
-
+# Directory to stage Zeek extracted files before processing
+redef FileExtract::prefix = "/nsm/zeek/extracted/";
+# Set a limit to the file size
+redef FileExtract::default_limit = 9000000;
+# These are the mimetypes we want to rip off the networks
+export {
+ global _mime_whitelist: table[string] of string = {
+ ["application/x-dosexec"] = "exe",
+ ["application/pdf"] = "pdf",
+ ["application/msword"] = "doc",
+ ["application/vnd.ms-powerpoint"] = "doc",
+ ["application/rtf"] = "doc",
+ ["application/vnd.ms-word.document.macroenabled.12"] = "doc",
+ ["application/vnd.ms-word.template.macroenabled.12"] = "doc",
+ ["application/vnd.ms-powerpoint.template.macroenabled.12"] = "doc",
+ ["application/vnd.ms-excel"] = "doc",
+ ["application/vnd.ms-excel.addin.macroenabled.12"] = "doc",
+ ["application/vnd.ms-excel.sheet.binary.macroenabled.12"] = "doc",
+ ["application/vnd.ms-excel.template.macroenabled.12"] = "doc",
+ ["application/vnd.ms-excel.sheet.macroenabled.12"] = "doc",
+ ["application/vnd.openxmlformats-officedocument.presentationml.presentation"] = "doc",
+ ["application/vnd.openxmlformats-officedocument.presentationml.slide"] = "doc",
+ ["application/vnd.openxmlformats-officedocument.presentationml.slideshow"] = "doc",
+ ["application/vnd.openxmlformats-officedocument.presentationml.template"] = "doc",
+ ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"] = "doc",
+ ["application/vnd.openxmlformats-officedocument.spreadsheetml.template"] = "doc",
+ ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"] = "doc",
+ ["application/vnd.openxmlformats-officedocument.wordprocessingml.template"] = "doc",
+ ["application/vnd.ms-powerpoint.addin.macroenabled.12"] = "doc",
+ ["application/vnd.ms-powerpoint.slide.macroenabled.12"] = "doc",
+ ["application/vnd.ms-powerpoint.presentation.macroenabled.12"] = "doc",
+ ["application/vnd.ms-powerpoint.slideshow.macroenabled.12"] = "doc",
+ ["application/vnd.openxmlformats-officedocument"] = "doc"
+ # Need to add other types such as zip, ps1, etc
+ };
+}
+# Start grabbing the file from the network if it matches the mimetype
+event file_sniff(f: fa_file, meta: fa_metadata) &priority=10 {
local ext = "";
-
- if ( meta?$mime_type )
- ext = ext_map[meta$mime_type];
-
- local fname = fmt("/nsm/zeek/extracted/%s-%s.%s", f$source, f$id, ext);
- Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
+ if( meta?$mime_type ) {
+ if ( meta$mime_type !in _mime_whitelist ) {
+ return;
}
+ ext = _mime_whitelist[meta$mime_type];
+ local fname = fmt("%s-%s.%s", f$source, f$id, ext);
+ Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
+ }
+}
+# Wait for file_state_remove before you do anything. This is when it is actually done.
+event file_state_remove(f: fa_file)
+ {
+ if ( !f$info?$extracted || FileExtract::prefix == "" ) {
+ return;
+ }
+ # Check some conditions so we know the file is intact:
+ # Check for MD5
+ # Check for total_bytes
+ # Check for missing bytes
+ # Check if timed out
+ if ( !f$info?$md5 || !f?$total_bytes || f$missing_bytes > 0 || f$info$timedout) {
+ # Delete the file if it didn't pass our requirements check.
+
+ local nuke = fmt("rm %s/%s", FileExtract::prefix, f$info$extracted);
+ when ( local nukeit = Exec::run([$cmd=nuke]) )
+ {
+ }
+ return;
+ }
+ local orig = f$info$extracted;
+ local split_orig = split_string(f$info$extracted, /\./);
+ local extension = split_orig[|split_orig|-1];
+ local dest = fmt("%scomplete/%s-%s-%s.%s", FileExtract::prefix, f$source, f$id, f$info$md5, extension);
+ # Copy it to the $prefix/complete folder then delete it. I got some weird results with moving when it came to watchdog in python.
+ local cmd = fmt("cp %s/%s %s && rm %s/%s", FileExtract::prefix, orig, dest, FileExtract::prefix, orig);
+ when ( local result = Exec::run([$cmd=cmd]) )
+ {
+ }
+ f$info$extracted = dest;
+ }
+