diff --git a/pillar/logstash/eval.sls b/pillar/logstash/eval.sls index 7f817ed39..f6cf222b3 100644 --- a/pillar/logstash/eval.sls +++ b/pillar/logstash/eval.sls @@ -25,3 +25,4 @@ logstash: - so/logstash-ossec-template.json - so/logstash-strelka-template.json - so/logstash-template.json + - so/logstash-bro-template.json diff --git a/salt/common/tools/sbin/so-auth-restart b/salt/common/tools/sbin/so-auth-restart old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-auth-start b/salt/common/tools/sbin/so-auth-start old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-auth-stop b/salt/common/tools/sbin/so-auth-stop old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-nodered-restart b/salt/common/tools/sbin/so-nodered-restart new file mode 100755 index 000000000..b30e7d4bb --- /dev/null +++ b/salt/common/tools/sbin/so-nodered-restart @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart nodered $1 diff --git a/salt/common/tools/sbin/so-nodered-start b/salt/common/tools/sbin/so-nodered-start new file mode 100755 index 000000000..5d21022cf --- /dev/null +++ b/salt/common/tools/sbin/so-nodered-start @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start nodered $1 + diff --git a/salt/common/tools/sbin/so-nodered-stop b/salt/common/tools/sbin/so-nodered-stop new file mode 100755 index 000000000..9a83d6cef --- /dev/null +++ b/salt/common/tools/sbin/so-nodered-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop nodered $1 diff --git a/salt/hive/thehive/etc/application.conf b/salt/hive/thehive/etc/application.conf index 14a635e54..ae588a42d 100644 --- a/salt/hive/thehive/etc/application.conf +++ b/salt/hive/thehive/etc/application.conf @@ -209,7 +209,10 @@ misp { #} ## <-- Uncomment to complete the configuration } webhooks { - SOCtopusWebHook { - url = "http://{{ MASTERIP }}:7000/enrich" + NodeRedWebHook { + url = "http://{{ MASTERIP }}:1880/thehive" } + #SOCtopusWebHook { + # url = "http://{{ MASTERIP }}:7000/enrich" + #} } diff --git a/salt/logstash/pipelines/config/so/9000_output_bro.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_bro.conf.jinja index 553500281..acc31ae00 100644 --- a/salt/logstash/pipelines/config/so/9000_output_bro.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_bro.conf.jinja @@ -23,8 +23,8 @@ output { pipeline => "%{event_type}" hosts => "{{ ES }}" index => "logstash-bro-%{+YYYY.MM.dd}" - template_name => "logstash" - template => "/logstash-template.json" + template_name => "logstash-bro" + template => "/logstash-bro-template.json" template_overwrite => true } } diff --git a/salt/nodered/files/nodered_load_flows b/salt/nodered/files/nodered_load_flows new file mode 100644 index 000000000..c48fcd692 --- /dev/null +++ b/salt/nodered/files/nodered_load_flows @@ -0,0 +1,11 @@ +{%- set ip = salt['pillar.get']('static:masterip', '') -%} +#!/bin/bash + +echo "Waiting for connection" +until $(curl --output /dev/null --silent --head http://{{ ip }}:1880); do + echo '.' + sleep 1 +done +echo "Loading flows..." +curl -XPOST -v -H "Content-Type: application/json" -d @/opt/so/saltstack/salt/nodered/so_flows.json {{ ip }}:1880/flows +echo "Done loading..." diff --git a/salt/nodered/files/so_flows.json b/salt/nodered/files/so_flows.json new file mode 100644 index 000000000..8ab8cbf81 --- /dev/null +++ b/salt/nodered/files/so_flows.json @@ -0,0 +1,4 @@ +{%- set MASTERIP = salt['pillar.get']('static:masterip', '') -%} +{%- set HIVEKEY = salt['pillar.get']('static:hivekey', '') -%} +{%- set CORTEXKEY = salt['pillar.get']('static:cortexkey', '') -%} +[{"id":"dca608c3.7d8af8","type":"tab","label":"TheHive - Webhook Events","disabled":false,"info":""},{"id":"4db74fa6.2556d","type":"tls-config","z":"","name":"","cert":"","key":"","ca":"","certname":"","keyname":"","caname":"","servername":"","verifyservercert":false},{"id":"aa6cf50d.a02fc8","type":"http in","z":"dca608c3.7d8af8","name":"TheHive Listener","url":"/thehive","method":"post","upload":false,"swaggerDoc":"","x":120,"y":780,"wires":[["2b92aebb.853dc2","2fce29bb.1b1376","82ad0f08.7a53f"]]},{"id":"2b92aebb.853dc2","type":"debug","z":"dca608c3.7d8af8","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","x":470,"y":940,"wires":[]},{"id":"a4ecb84a.805958","type":"switch","z":"dca608c3.7d8af8","name":"Operation","property":"payload.operation","propertyType":"msg","rules":[{"t":"eq","v":"Creation","vt":"str"},{"t":"eq","v":"Update","vt":"str"},{"t":"eq","v":"Delete","vt":"str"}],"checkall":"false","repair":false,"outputs":3,"x":580,"y":780,"wires":[["f1e954fd.3c21d8"],["65928861.c90a48"],["a259a26c.a21"]],"outputLabels":["Creation","Update","Delete"]},{"id":"f1e954fd.3c21d8","type":"switch","z":"dca608c3.7d8af8","name":"Creation","property":"payload.objectType","propertyType":"msg","rules":[{"t":"eq","v":"case","vt":"str"},{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"case_task","vt":"str"},{"t":"eq","v":"case_task_log","vt":"str"},{"t":"eq","v":"case_artifact_job","vt":"str"},{"t":"eq","v":"alert","vt":"str"},{"t":"eq","v":"user","vt":"str"}],"checkall":"false","repair":false,"outputs":7,"x":900,"y":480,"wires":[["e88b4cc2.f6afe"],["8c54e39.a1b4f2"],["64203fe8.e0ad5"],["3511de51.889a02"],["14544a8b.b6b2f5"],["44c595a4.45d45c"],["3eb4bedf.6e20a2"]],"inputLabels":["Operation"],"outputLabels":["case","case_artifact","case_task","case_task_log","action","alert","user"],"info":"No webhook data is received for the following events:\n\n- Creation of Dashboard\n- Creation of Case Templates\n"},{"id":"65928861.c90a48","type":"switch","z":"dca608c3.7d8af8","name":"Update","property":"payload.objectType","propertyType":"msg","rules":[{"t":"eq","v":"case","vt":"str"},{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"case_artifact_job","vt":"str"},{"t":"eq","v":"case_task","vt":"str"},{"t":"eq","v":"case_task_log","vt":"str"},{"t":"eq","v":"alert","vt":"str"},{"t":"eq","v":"user","vt":"str"}],"checkall":"false","repair":false,"outputs":7,"x":900,"y":860,"wires":[["eebe1748.1cd348"],["d703adc0.12fd1"],["2b738415.408d4c"],["6d97371a.406348"],["4ae621e1.9ae6"],["5786cee2.98109"],["54077728.447648"]],"inputLabels":["Operation"],"outputLabels":["case","case_artifact",null,"case_task","case_task_log","alert","user"]},{"id":"a259a26c.a21","type":"switch","z":"dca608c3.7d8af8","name":"Delete","property":"payload.objectType","propertyType":"msg","rules":[{"t":"eq","v":"case","vt":"str"},{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"case_task_log","vt":"str"}],"checkall":"false","repair":false,"outputs":3,"x":890,"y":1200,"wires":[["60c8bcfb.eff1f4"],["df708bab.348308"],["e9a8650c.e20cc8"]],"outputLabels":["case","case_artifact",""],"info":"Deleting a case task doesnt actually trigger a delete event. It triggers an `update` event where the status = cancelled"},{"id":"54077728.447648","type":"switch","z":"dca608c3.7d8af8","name":"User","property":"payload.object.status","propertyType":"msg","rules":[{"t":"eq","v":"Locked","vt":"str"},{"t":"eq","v":"Ok","vt":"str"}],"checkall":"false","repair":false,"outputs":2,"x":1130,"y":980,"wires":[["9429d6c5.5ac788"],["4e3e091c.d35388"]]},{"id":"9429d6c5.5ac788","type":"function","z":"dca608c3.7d8af8","name":"status: Locked","func":"msg.topic = \"[The Hive] A user account was locked\";\nmsg.from = \"from@example.com\";\nmsg.to = \"to@example.com\";\nreturn msg;","outputs":1,"noerr":0,"x":1380,"y":972,"wires":[[]],"info":"- User account was locked"},{"id":"4e3e091c.d35388","type":"function","z":"dca608c3.7d8af8","name":"status: Ok","func":"msg.topic = \"[The Hive] A user account was changed\";\nmsg.from = \"from@example.com\";\nmsg.to = \"to@example.com\";\nreturn msg;","outputs":1,"noerr":0,"x":1360,"y":1020,"wires":[[]],"info":"- User account was unlocked\n- User description was changed\n- User role was changed\n- User API key was added\n- User API key was revoked\n"},{"id":"485f3be.1ffcfc4","type":"function","z":"dca608c3.7d8af8","name":"status: Open","func":"// Fires when a Case is updated AND status = open\n// This can include things like TLP/PAP changes\n\nreturn msg;","outputs":1,"noerr":0,"x":1370,"y":660,"wires":[[]]},{"id":"eebe1748.1cd348","type":"switch","z":"dca608c3.7d8af8","name":"case","property":"payload.object.status","propertyType":"msg","rules":[{"t":"eq","v":"Open","vt":"str"}],"checkall":"true","repair":false,"outputs":1,"x":1130,"y":740,"wires":[["485f3be.1ffcfc4","e4b7b4bf.2fb828"]],"info":"- A case was modified"},{"id":"8c54e39.a1b4f2","type":"switch","z":"dca608c3.7d8af8","name":"case_artifact: Run Analyzer","property":"payload.object.dataType","propertyType":"msg","rules":[{"t":"eq","v":"ip","vt":"str"},{"t":"eq","v":"domain","vt":"str"}],"checkall":"true","repair":false,"outputs":2,"x":1600,"y":340,"wires":[["eb8cfeb7.a7118","a5dd8a8a.065b88"],["eb8cfeb7.a7118","a5dd8a8a.065b88"]],"info":"# References\n\n\n"},{"id":"2fce29bb.1b1376","type":"function","z":"dca608c3.7d8af8","name":"Add headers","func":"msg.thehive_url = 'https://{{ MASTERIP }}/thehive';\nmsg.cortex_url = 'https://{{ MASTERIP }}/cortex';\nmsg.cortex_id = 'CORTEX-SERVER-ID';\nreturn msg;","outputs":1,"noerr":0,"x":350,"y":780,"wires":[["a4ecb84a.805958"]]},{"id":"e4b7b4bf.2fb828","type":"function","z":"dca608c3.7d8af8","name":"status: Resolved","func":"// Fires when a case is closed (resolved)\n\nreturn msg;","outputs":1,"noerr":0,"x":1390,"y":720,"wires":[[]]},{"id":"e88b4cc2.f6afe","type":"function","z":"dca608c3.7d8af8","name":"case","func":"// Fires when a case is created\n// or when a responder is generated against a case\n\nreturn msg;","outputs":1,"noerr":0,"x":1130,"y":320,"wires":[[]]},{"id":"64203fe8.e0ad5","type":"function","z":"dca608c3.7d8af8","name":"case_task","func":"// Fires when a case task is created\nreturn msg;","outputs":1,"noerr":0,"x":1140,"y":400,"wires":[[]]},{"id":"3511de51.889a02","type":"function","z":"dca608c3.7d8af8","name":"case_task_log","func":"// Fires when a case task log is created\n\nreturn msg;","outputs":1,"noerr":0,"x":1163,"y":440,"wires":[[]]},{"id":"14544a8b.b6b2f5","type":"function","z":"dca608c3.7d8af8","name":"case_artifact_job","func":"// Fires when a Responder or Analyzser is Run on an existing observable\n\nreturn msg;","outputs":1,"noerr":0,"x":1173,"y":480,"wires":[[]]},{"id":"2b738415.408d4c","type":"function","z":"dca608c3.7d8af8","name":"case_artifact_job","func":"\nreturn msg;","outputs":1,"noerr":0,"x":1170,"y":820,"wires":[[]]},{"id":"3eb4bedf.6e20a2","type":"function","z":"dca608c3.7d8af8","name":"user","func":"// Fires when a user is created\n\nreturn msg;","outputs":1,"noerr":0,"x":1133,"y":560,"wires":[[]]},{"id":"d703adc0.12fd1","type":"function","z":"dca608c3.7d8af8","name":"case_artifact","func":"// Fires when an artifact is updated\nreturn msg;","outputs":1,"noerr":0,"x":1150,"y":780,"wires":[[]]},{"id":"6d97371a.406348","type":"function","z":"dca608c3.7d8af8","name":"case_task","func":"// Fires when a case task is updated\nreturn msg;","outputs":1,"noerr":0,"x":1140,"y":860,"wires":[[]]},{"id":"4ae621e1.9ae6","type":"function","z":"dca608c3.7d8af8","name":"case_task_log","func":"//Fires when a case_task_log is updated\n\nreturn msg;","outputs":1,"noerr":0,"x":1160,"y":900,"wires":[[]]},{"id":"60c8bcfb.eff1f4","type":"function","z":"dca608c3.7d8af8","name":"case","func":"//Fires when a case is deleted\nreturn msg;","outputs":1,"noerr":0,"x":1130,"y":1160,"wires":[[]]},{"id":"df708bab.348308","type":"function","z":"dca608c3.7d8af8","name":"case_artifact","func":"//Fires when a case_artifact is deleted\nreturn msg;","outputs":1,"noerr":0,"x":1150,"y":1200,"wires":[[]]},{"id":"e9a8650c.e20cc8","type":"function","z":"dca608c3.7d8af8","name":"case_task_log","func":"//Fires when a case_task_log is deleted\nreturn msg;","outputs":1,"noerr":0,"x":1160,"y":1240,"wires":[[]]},{"id":"5786cee2.98109","type":"function","z":"dca608c3.7d8af8","name":"alert","func":"//Fires when an alert is updated\nreturn msg;","outputs":1,"noerr":0,"x":1130,"y":940,"wires":[[]]},{"id":"44c595a4.45d45c","type":"change","z":"dca608c3.7d8af8","d":true,"name":"Convert Alert Msg to Artifacts","rules":[{"t":"move","p":"payload.object.artifacts","pt":"msg","to":"payload","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":1200,"y":520,"wires":[["6dcca25e.04bd2c"]]},{"id":"6dcca25e.04bd2c","type":"split","z":"dca608c3.7d8af8","name":"Split Artifacts","splt":"\\n","spltType":"str","arraySplt":1,"arraySpltType":"len","stream":false,"addname":"","x":1430,"y":520,"wires":[["767c84f2.c9ba2c"]]},{"id":"767c84f2.c9ba2c","type":"switch","z":"dca608c3.7d8af8","name":"alert: Run Analyzer","property":"payload.dataType","propertyType":"msg","rules":[{"t":"eq","v":"ip","vt":"str"},{"t":"eq","v":"domain","vt":"str"}],"checkall":"true","repair":false,"outputs":2,"x":1630,"y":400,"wires":[["eb8cfeb7.a7118","a5dd8a8a.065b88"],["a5dd8a8a.065b88","eb8cfeb7.a7118"]],"info":"# References\n\n\n"},{"id":"82ad0f08.7a53f","type":"http response","z":"dca608c3.7d8af8","name":"Ack Event Receipt","statusCode":"200","headers":{},"x":250,"y":940,"wires":[]},{"id":"a5dd8a8a.065b88","type":"function","z":"dca608c3.7d8af8","name":"Run Analyzer: CERT DNS","func":"msg.analyzer_id = \"4f28afc20d78f98df425e36e561af33f\";\n\nif (msg.payload.objectId) {\n msg.tag = \"case_artifact\"\n msg.artifact_id = msg.payload.objectId\n msg.url = msg.thehive_url + '/api/connector/cortex/job';\n msg.payload = {\n 'cortexId' : msg.cortex_id,\n 'artifactId': msg.artifact_id,\n 'analyzerId': msg.analyzer_id\n };\n}\nelse {\n msg.tag = \"observable\"\n msg.observable = msg.payload.data\n msg.dataType = msg.payload.dataType\n\n msg.url = msg.cortex_url + '/api/analyzer/' + msg.analyzer_id + '/run';\n msg.payload = {\n 'data' : msg.observable,\n 'dataType': msg.dataType \n };\n}\nreturn msg;","outputs":1,"noerr":0,"x":1930,"y":420,"wires":[["f050a09f.b2201"]]},{"id":"eb8cfeb7.a7118","type":"function","z":"dca608c3.7d8af8","name":"Run Analyzer: Urlscan","func":"msg.analyzer_id = \"54e51b62c6c8ddc3cbc3cbdd889a0557\";\n\nif (msg.payload.objectId) {\n msg.tag = \"case_artifact\"\n msg.artifact_id = msg.payload.objectId\n msg.url = msg.thehive_url + '/api/connector/cortex/job';\n msg.payload = {\n 'cortexId' : msg.cortex_id,\n 'artifactId': msg.artifact_id,\n 'analyzerId': msg.analyzer_id\n };\n}\nelse {\n msg.tag = \"observable\"\n msg.observable = msg.payload.data\n msg.dataType = msg.payload.dataType\n\n msg.url = msg.cortex_url + '/api/analyzer/' + msg.analyzer_id + '/run';\n msg.payload = {\n 'data' : msg.observable,\n 'dataType': msg.dataType \n };\n}\nreturn msg;","outputs":1,"noerr":0,"x":1920,"y":320,"wires":[["f050a09f.b2201"]]},{"id":"1c448528.3032fb","type":"http request","z":"dca608c3.7d8af8","name":"Submit to Cortex","method":"POST","ret":"obj","paytoqs":false,"url":"","tls":"4db74fa6.2556d","persist":false,"proxy":"","authType":"bearer","credentials": {"user": "", "password": "{{ CORTEXKEY }}"},"x":2450,"y":420,"wires":[["ea6614fb.752a78"]]},{"id":"ea6614fb.752a78","type":"debug","z":"dca608c3.7d8af8","name":"Debug","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","x":2670,"y":360,"wires":[]},{"id":"f050a09f.b2201","type":"switch","z":"dca608c3.7d8af8","name":"Cases vs Alerts","property":"tag","propertyType":"msg","rules":[{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"observable","vt":"str"}],"checkall":"true","repair":false,"outputs":2,"x":2200,"y":360,"wires":[["f7fca977.a73b28"],["1c448528.3032fb"]],"inputLabels":["Data"],"outputLabels":["Cases","Alerts"]},{"id":"f7fca977.a73b28","type":"http request","z":"dca608c3.7d8af8","name":"Submit to TheHive","method":"POST","ret":"obj","paytoqs":false,"url":"","tls":"4db74fa6.2556d","persist":false,"proxy":"","authType":"bearer","credentials": {"user": "", "password": "{{ HIVEKEY }}"},"x":2450,"y":280,"wires":[["ea6614fb.752a78"]]}] diff --git a/salt/nodered/init.sls b/salt/nodered/init.sls index 96aee2630..af58ab866 100644 --- a/salt/nodered/init.sls +++ b/salt/nodered/init.sls @@ -34,6 +34,23 @@ nodered: # - mode: 775 # - makedirs: True +noderedflows: + file.recurse: + - name: /opt/so/saltstack/salt/nodered/ + - source: salt://nodered/files + - user: 947 + - group: 939 + - template: jinja + +noderedflowsload: + file.managed: + - name: /usr/sbin/so-nodered-load-flows + - source: salt://nodered/files/nodered_load_flows + - user: 0 + - group: 0 + - mode: 755 + - template: jinja + noderedlog: file.directory: - name: /opt/so/log/nodered @@ -44,10 +61,15 @@ noderedlog: so-nodered: docker_container.running: - - image: soshybridhunter/so-nodered:HH1.1.5 + - image: soshybridhunter/so-nodered:HH1.2.1 - interactive: True - binds: - /opt/so/conf/nodered/:/data:rw - port_bindings: - 0.0.0.0:1880:1880 +so-nodered-flows: + cmd.run: + - name: /usr/sbin/so-nodered-load-flows + - cwd: / + diff --git a/salt/playbook/init.sls b/salt/playbook/init.sls index 049c5fab9..04c321ab7 100644 --- a/salt/playbook/init.sls +++ b/salt/playbook/init.sls @@ -52,3 +52,10 @@ so-navigator: - identifier: so-playbook-sync - user: root - minute: '*/5' + +/usr/sbin/so-playbook-ruleupdate: + cron.present: + - identifier: so-playbook-ruleupdate + - user: root + - minute: '1' + - hour: '6' diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index db54bf62e..88a1ecbde 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -48,15 +48,9 @@ zeekextractdir: - group: 939 - makedirs: True -zeeksfafincompletedir: +zeekextractcompletedir: file.directory: - - name: /nsm/faf/files/incomplete - - user: 937 - - makedirs: true - -zeeksfafcompletedir: - file.directory: - - name: /nsm/faf/files/complete + - name: /nsm/zeek/extracted/complete - user: 937 - makedirs: true diff --git a/salt/zeek/policy/securityonion/file-extraction/extract.zeek b/salt/zeek/policy/securityonion/file-extraction/extract.zeek index b8e8478bd..fbb635982 100644 --- a/salt/zeek/policy/securityonion/file-extraction/extract.zeek +++ b/salt/zeek/policy/securityonion/file-extraction/extract.zeek @@ -1,21 +1,80 @@ -global ext_map: table[string] of string = { - ["application/x-dosexec"] = "exe", - ["text/plain"] = "txt", - ["image/jpeg"] = "jpg", - ["image/png"] = "png", - ["text/html"] = "html", -} &default =""; - -event file_sniff(f: fa_file, meta: fa_metadata) - { - if ( ! meta?$mime_type || meta$mime_type != "application/x-dosexec" ) - return; - +# Directory to stage Zeek extracted files before processing +redef FileExtract::prefix = "/nsm/zeek/extracted/"; +# Set a limit to the file size +redef FileExtract::default_limit = 9000000; +# These are the mimetypes we want to rip off the networks +export { + global _mime_whitelist: table[string] of string = { + ["application/x-dosexec"] = "exe", + ["application/pdf"] = "pdf", + ["application/msword"] = "doc", + ["application/vnd.ms-powerpoint"] = "doc", + ["application/rtf"] = "doc", + ["application/vnd.ms-word.document.macroenabled.12"] = "doc", + ["application/vnd.ms-word.template.macroenabled.12"] = "doc", + ["application/vnd.ms-powerpoint.template.macroenabled.12"] = "doc", + ["application/vnd.ms-excel"] = "doc", + ["application/vnd.ms-excel.addin.macroenabled.12"] = "doc", + ["application/vnd.ms-excel.sheet.binary.macroenabled.12"] = "doc", + ["application/vnd.ms-excel.template.macroenabled.12"] = "doc", + ["application/vnd.ms-excel.sheet.macroenabled.12"] = "doc", + ["application/vnd.openxmlformats-officedocument.presentationml.presentation"] = "doc", + ["application/vnd.openxmlformats-officedocument.presentationml.slide"] = "doc", + ["application/vnd.openxmlformats-officedocument.presentationml.slideshow"] = "doc", + ["application/vnd.openxmlformats-officedocument.presentationml.template"] = "doc", + ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"] = "doc", + ["application/vnd.openxmlformats-officedocument.spreadsheetml.template"] = "doc", + ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"] = "doc", + ["application/vnd.openxmlformats-officedocument.wordprocessingml.template"] = "doc", + ["application/vnd.ms-powerpoint.addin.macroenabled.12"] = "doc", + ["application/vnd.ms-powerpoint.slide.macroenabled.12"] = "doc", + ["application/vnd.ms-powerpoint.presentation.macroenabled.12"] = "doc", + ["application/vnd.ms-powerpoint.slideshow.macroenabled.12"] = "doc", + ["application/vnd.openxmlformats-officedocument"] = "doc" + # Need to add other types such as zip, ps1, etc + }; +} +# Start grabbing the file from the network if it matches the mimetype +event file_sniff(f: fa_file, meta: fa_metadata) &priority=10 { local ext = ""; - - if ( meta?$mime_type ) - ext = ext_map[meta$mime_type]; - - local fname = fmt("/nsm/zeek/extracted/%s-%s.%s", f$source, f$id, ext); - Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]); + if( meta?$mime_type ) { + if ( meta$mime_type !in _mime_whitelist ) { + return; } + ext = _mime_whitelist[meta$mime_type]; + local fname = fmt("%s-%s.%s", f$source, f$id, ext); + Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]); + } +} +# Wait for file_state_remove before you do anything. This is when it is actually done. +event file_state_remove(f: fa_file) + { + if ( !f$info?$extracted || FileExtract::prefix == "" ) { + return; + } + # Check some conditions so we know the file is intact: + # Check for MD5 + # Check for total_bytes + # Check for missing bytes + # Check if timed out + if ( !f$info?$md5 || !f?$total_bytes || f$missing_bytes > 0 || f$info$timedout) { + # Delete the file if it didn't pass our requirements check. + + local nuke = fmt("rm %s/%s", FileExtract::prefix, f$info$extracted); + when ( local nukeit = Exec::run([$cmd=nuke]) ) + { + } + return; + } + local orig = f$info$extracted; + local split_orig = split_string(f$info$extracted, /\./); + local extension = split_orig[|split_orig|-1]; + local dest = fmt("%scomplete/%s-%s-%s.%s", FileExtract::prefix, f$source, f$id, f$info$md5, extension); + # Copy it to the $prefix/complete folder then delete it. I got some weird results with moving when it came to watchdog in python. + local cmd = fmt("cp %s/%s %s && rm %s/%s", FileExtract::prefix, orig, dest, FileExtract::prefix, orig); + when ( local result = Exec::run([$cmd=cmd]) ) + { + } + f$info$extracted = dest; + } +