From 762a3bea177a021dc97a1dcf9c1771d9c8bcc359 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 25 Jan 2024 09:59:26 -0500 Subject: [PATCH 01/19] Defaults and Annotations --- salt/suricata/defaults.yaml | 10 +++++++ salt/suricata/soc_suricata.yaml | 50 +++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index e9e39d40a..4961ae50a 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -128,6 +128,16 @@ suricata: enabled: "no" pcap-log: enabled: "no" + compression: "none" + lz4-checksum: "no" + lz4-level: 8 + filename: "%n/so-pcap.%t" + limit: "1000mb" + mode: "multi" + max-files: 10 + use-stream-depth: "no" + conditional: "all" + dir: "/nsm/pcap" alert-debug: enabled: "no" alert-prelude: diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 30f277c0a..58a2273b9 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -153,6 +153,53 @@ suricata: header: description: Header name where the actual IP address will be reported. helpLink: suricata.html + pcap-log: + enabled: + description: Enable Suricata to collect PCAP. + helpLink: suricata.html + compression: + description: Enable compression of Suricata PCAP. Currently unsupported + advanced: True + readonly: True + helpLink: suricata.html + lz4-checksum: + description: Enable PCAP lz4 checksum. Currently unsupported + advanced: True + readonly: True + helpLink: suricata.html + lz4-level: + description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported + advanced: True + readonly: True + helpLink: suricata.html + filename: + description: Filename output for Suricata PCAP. + advanced: True + readonly: True + helpLink: suricata.html + limit: + description: File size limit per thread. To determine max PCAP size multiple threads x max-files x limit. + helpLink: suricata.html + mode: + description: Suricata PCAP mode. Currenlty only multi is supported. + advanced: True + readonly: True + helpLink: suricata.html + max-files: + description: Max PCAP files per thread. To determine max PCAP size multiple threads x max-files x limit. + helpLink: suricata.html + use-stream-depth: + description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. + advanced: True + helpLink: suricata.html + conditional: + description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules. + helpLink: suricata.html + dir: + description: Parent directory to store PCAP. + advanced: True + readonly: True + helpLink: suricata.html asn1-max-frames: description: Maximum nuber of asn1 frames to decode. helpLink: suricata.html @@ -209,6 +256,9 @@ suricata: memcap: description: Can be specified in kb,mb,gb. helpLink: suricata.html + depth: + description: Controls how far into a stream that reassembly is done. + helpLink: suricata.html host: hash-size: description: Hash size in bytes. From 1a2245a1ed188be34b2ffbcb6304567c9c5abdee Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 13:44:53 -0500 Subject: [PATCH 02/19] Add so-minion modifications --- salt/manager/tools/sbin/so-minion | 25 +++++++++++++++++++++++++ salt/suricata/soc_suricata.yaml | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index d5225cc82..12349b680 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -79,6 +79,30 @@ function getinstallinfo() { source <(echo $INSTALLVARS) } +function pcapspace() { + + local NSMSIZE=$(salt \* disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') + local ROOTSIZE=$(salt \* disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') + + if [[ "$NSMSIZE" == "null" ]]; then + # Looks like there is no dedicated nsm partition. Using root + local SPACESIZE=$ROOTSIZE + else + local SPACESIZE=$NSMSIZE + fi + + local s=$(( $SPACESIZE / 1000000 )) + local s1=$(( $s / 2 )) + local s2=$(( $s1 / $lb_procs )) + + printf '%s\n'\ + "suricata:"\ + " config:"\ + " output:"\ + " pcap-log: $s" >> $PILLARFILE + +} + function testMinion() { # Always run on the host, since this is going to be the manager of a distributed grid, or an eval/standalone. # Distributed managers must run this in order for the sensor nodes to have access to the so-tcpreplay image. @@ -252,6 +276,7 @@ function add_sensor_to_minion() { if [[ $is_pcaplimit ]]; then echo " config:" >> $PILLARFILE echo " diskfreepercentage: 60" >> $PILLARFILE + pcapspace fi echo " " >> $PILLARFILE } diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 58a2273b9..5dddd7442 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -181,7 +181,7 @@ suricata: description: File size limit per thread. To determine max PCAP size multiple threads x max-files x limit. helpLink: suricata.html mode: - description: Suricata PCAP mode. Currenlty only multi is supported. + description: Suricata PCAP mode. Currently only multi is supported. advanced: True readonly: True helpLink: suricata.html From 5b05aec96aaa9cbf73294bce24fed2d59f5e4be4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 14:56:51 -0500 Subject: [PATCH 03/19] Target sspecific minion --- salt/manager/tools/sbin/so-minion | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 12349b680..877796620 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -81,8 +81,8 @@ function getinstallinfo() { function pcapspace() { - local NSMSIZE=$(salt \* disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') - local ROOTSIZE=$(salt \* disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') + local NSMSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') + local ROOTSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') if [[ "$NSMSIZE" == "null" ]]; then # Looks like there is no dedicated nsm partition. Using root From 0c969312e2ee61eed8e6c0d571e44dc7ee7cdcec Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 15:22:20 -0500 Subject: [PATCH 04/19] Add Globals --- salt/global/defaults.yaml | 2 ++ salt/global/soc_global.yaml | 5 +++++ salt/manager/tools/sbin/so-minion | 3 ++- salt/sensoroni/files/sensoroni.json | 11 ++++++++++- salt/suricata/soc_suricata.yaml | 4 +++- 5 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 salt/global/defaults.yaml diff --git a/salt/global/defaults.yaml b/salt/global/defaults.yaml new file mode 100644 index 000000000..bd7244a58 --- /dev/null +++ b/salt/global/defaults.yaml @@ -0,0 +1,2 @@ +global: + pcapengine: STENO \ No newline at end of file diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index 14d637d50..fc1c09b1c 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -14,6 +14,11 @@ global: regex: ^(ZEEK|SURICATA)$ regexFailureMessage: You must enter either ZEEK or SURICATA. global: True + pcapengine: + description: What engine to use for generating pcap. Options are STENO and SURICATA. + regex: ^(STENO|SURICATA)$ + regexFailureMessage: You must enter either STENO or SURICATA. + global: True ids: description: Which IDS engine to use. Currently only Suricata is supported. global: True diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 877796620..4995e1c9d 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -99,7 +99,8 @@ function pcapspace() { "suricata:"\ " config:"\ " output:"\ - " pcap-log: $s" >> $PILLARFILE + " pcap-log:"\ + " max-files: $s" >> $PILLARFILE } diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 59ce500e3..c5608ba56 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -23,13 +23,22 @@ "importer": {}, "statickeyauth": { "apiKey": "{{ GLOBALS.sensoroni_key }}" -{%- if PCAPMERGED.enabled %} +{%- if PCAPMERGED.enabled %} +{%- if PCAPENGINE.steno %} }, "stenoquery": { "executablePath": "/opt/sensoroni/scripts/stenoquery.sh", "pcapInputPath": "/nsm/pcap", "pcapOutputPath": "/nsm/pcapout" } +{%- elif PCAPENGINE.suri %} + }, + "suriquery": { + "executablePath": "/opt/sensoroni/scripts/suriquery.sh", + "pcapInputPath": "/nsm/suripcap", + "pcapOutputPath": "/nsm/pcapout" + } +{%- endif %} {%- else %} } {%- endif %} diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 5dddd7442..7153eb9a1 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -155,8 +155,10 @@ suricata: helpLink: suricata.html pcap-log: enabled: - description: Enable Suricata to collect PCAP. + description: This value is ignored by SO. pcapengine in globals takes predidence. + readonly: True helpLink: suricata.html + advanced: True compression: description: Enable compression of Suricata PCAP. Currently unsupported advanced: True From 88c01a22d6454b318c0c94a1718c93fb69b34500 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 15:27:28 -0500 Subject: [PATCH 05/19] Add annotation logic --- salt/suricata/soc_suricata.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 7153eb9a1..52352d043 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -44,6 +44,7 @@ suricata: set-cpu-affinity: description: Bind(yes) or unbind(no) management and worker threads to a core or range of cores. regex: ^(yes|no)$ + regexFailureMessage: You must enter either yes or no. helpLink: suricata.html cpu-affinity: management-cpu-set: @@ -155,7 +156,7 @@ suricata: helpLink: suricata.html pcap-log: enabled: - description: This value is ignored by SO. pcapengine in globals takes predidence. + description: This value is ignored by SO. pcapengine in globals takes precidence. readonly: True helpLink: suricata.html advanced: True @@ -193,9 +194,13 @@ suricata: use-stream-depth: description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. advanced: True + regex: ^(yes|no)$ + regexFailureMessage: You must enter either yes or no. helpLink: suricata.html conditional: description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules. + regex: ^(all|alert|tag)$ + regexFailureMessage: You must enter either all, alert or tag. helpLink: suricata.html dir: description: Parent directory to store PCAP. From ab551a747ddafe1fd0b602e3f84d8130e9ffe5bc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 15:44:57 -0500 Subject: [PATCH 06/19] Threads placeholder logic --- salt/suricata/enabled.sls | 3 +++ salt/suricata/pcap.sls | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 salt/suricata/pcap.sls diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index ce309e41a..6dce49c8c 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -12,6 +12,9 @@ include: - suricata.config - suricata.sostatus + if blah + - suricata.pcap + endif so-suricata: docker_container.running: diff --git a/salt/suricata/pcap.sls b/salt/suricata/pcap.sls new file mode 100644 index 000000000..f677532f0 --- /dev/null +++ b/salt/suricata/pcap.sls @@ -0,0 +1,25 @@ +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% import_yaml 'suricata/defaults.yaml' as SURICATADEFAULTS %} +{% set SURICATAMERGED = salt['pillar.get']('suricata', SURICATADEFAULTS.suricata, merge=True) %} + +suripcapdir: + file.directory: + - name: /nsm/suripcap + - user: 940 + - group: 939 + - mode: 755 + - makedirs: True + +{{ SURICATAMERGED.config['af-packet'].threads }} + +for thread in afp.threads + +suripcapthreaddir: + file.directory: + - name: /nsm/suripcap/{{thread}} + - user: 940 + - group: 939 + - mode: 755 + - makedirs: True + +endfor \ No newline at end of file From 88d2ddba8bddeeac28fbadf12c826c04a4a61e82 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 15:53:54 -0500 Subject: [PATCH 07/19] add placeholder for telegraf --- salt/telegraf/scripts/oldpcap.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index bb1be457f..d3f4b9a93 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -5,13 +5,18 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +{%- if pcap is steno +PCAPLOC=/host/nsm/pcap +{%- else %} +PCAPLOC=/host/nsm/suripcap +{%- endif %} # if this script isn't already running if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then # Get the data - OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) + OLDPCAP=$(find $PCAPLOC -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) DATE=$(date +%s) AGE=$(($DATE - $OLDPCAP)) From d118ff4728454cb08cd020234b6d858c5edc3656 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 29 Jan 2024 16:54:08 -0500 Subject: [PATCH 08/19] add GLOBALS.pcap_engine --- salt/global/map.jinja | 2 ++ salt/sensoroni/files/sensoroni.json | 6 +++--- salt/suricata/enabled.sls | 4 ++-- salt/suricata/pcap.sls | 13 +++++-------- salt/telegraf/config.sls | 2 ++ salt/telegraf/scripts/oldpcap.sh | 7 +++---- salt/vars/globals.map.jinja | 2 ++ 7 files changed, 19 insertions(+), 17 deletions(-) create mode 100644 salt/global/map.jinja diff --git a/salt/global/map.jinja b/salt/global/map.jinja new file mode 100644 index 000000000..54abb8c79 --- /dev/null +++ b/salt/global/map.jinja @@ -0,0 +1,2 @@ +{% import_yaml 'global/defaults.yaml' as GLOBALDEFAULTS %} +{% set GLOBALMERGED = salt['pillar.get']('global', GLOBALDEFAULTS.global, merge=True) %} diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index c5608ba56..f813dad2f 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -24,21 +24,21 @@ "statickeyauth": { "apiKey": "{{ GLOBALS.sensoroni_key }}" {%- if PCAPMERGED.enabled %} -{%- if PCAPENGINE.steno %} +{%- if GLOBALS.pcap_engine == "STENO" %} }, "stenoquery": { "executablePath": "/opt/sensoroni/scripts/stenoquery.sh", "pcapInputPath": "/nsm/pcap", "pcapOutputPath": "/nsm/pcapout" } -{%- elif PCAPENGINE.suri %} +{%- elif GLOBALS.pcap_engine == "SURICATA" %} }, "suriquery": { "executablePath": "/opt/sensoroni/scripts/suriquery.sh", "pcapInputPath": "/nsm/suripcap", "pcapOutputPath": "/nsm/pcapout" } -{%- endif %} +{%- endif %} {%- else %} } {%- endif %} diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index 6dce49c8c..cf871906b 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -12,9 +12,9 @@ include: - suricata.config - suricata.sostatus - if blah +{% if GLOBALS.pcap_engine == "SURICATA" %} - suricata.pcap - endif +{% endif %} so-suricata: docker_container.running: diff --git a/salt/suricata/pcap.sls b/salt/suricata/pcap.sls index f677532f0..7a00d8d6a 100644 --- a/salt/suricata/pcap.sls +++ b/salt/suricata/pcap.sls @@ -1,6 +1,5 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} -{% import_yaml 'suricata/defaults.yaml' as SURICATADEFAULTS %} -{% set SURICATAMERGED = salt['pillar.get']('suricata', SURICATADEFAULTS.suricata, merge=True) %} +{% from 'suricata/map.jinja' import SURICATAMERGED %} suripcapdir: file.directory: @@ -10,16 +9,14 @@ suripcapdir: - mode: 755 - makedirs: True -{{ SURICATAMERGED.config['af-packet'].threads }} -for thread in afp.threads +{% for i in range(1, SURICATAMERGED.config['af-packet'].threads) + 1) %} -suripcapthreaddir: +suripcapthread{{i}}dir: file.directory: - - name: /nsm/suripcap/{{thread}} + - name: /nsm/suripcap/{{i}} - user: 940 - group: 939 - mode: 755 - - makedirs: True -endfor \ No newline at end of file +{% endfor %} diff --git a/salt/telegraf/config.sls b/salt/telegraf/config.sls index 0711260b5..a35be55f5 100644 --- a/salt/telegraf/config.sls +++ b/salt/telegraf/config.sls @@ -41,6 +41,8 @@ tgraf_sync_script_{{script}}: - mode: 770 - template: jinja - source: salt://telegraf/scripts/{{script}} + - defaults: + GLOBALS: {{ GLOBALS }} {% endfor %} telegraf_sbin: diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index d3f4b9a93..b68e71539 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -5,13 +5,12 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- if pcap is steno -PCAPLOC=/host/nsm/pcap -{%- else %} +{%- if GLOBALS.pcap_engine == "SURICATA" %} PCAPLOC=/host/nsm/suripcap +{%- else %} +PCAPLOC=/host/nsm/pcap {%- endif %} - # if this script isn't already running if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then diff --git a/salt/vars/globals.map.jinja b/salt/vars/globals.map.jinja index 3265cde18..990aeb39b 100644 --- a/salt/vars/globals.map.jinja +++ b/salt/vars/globals.map.jinja @@ -1,5 +1,6 @@ {% import 'vars/init.map.jinja' as INIT %} {% from 'docker/docker.map.jinja' import DOCKER %} +{% from 'global/map.jinja' import GLOBALMERGED %} {% from 'vars/' ~ INIT.GRAINS.role.split('-')[1] ~ '.map.jinja' import ROLE_GLOBALS %} {# role is so-role so we have to split off the 'so' #} @@ -20,6 +21,7 @@ 'influxdb_host': INIT.PILLAR.global.influxdb_host, 'manager_ip': INIT.PILLAR.global.managerip, 'md_engine': INIT.PILLAR.global.mdengine, + 'pcap_engine': GLOBALMERGED.pcapengine 'pipeline': INIT.PILLAR.global.pipeline, 'so_version': INIT.PILLAR.global.soversion, 'so_docker_gateway': DOCKER.gateway, From 37dcb84a09d836ec1b772fe25c42051e7bfdf797 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 10:50:01 -0500 Subject: [PATCH 09/19] add missing comma --- salt/vars/globals.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/vars/globals.map.jinja b/salt/vars/globals.map.jinja index 990aeb39b..624173217 100644 --- a/salt/vars/globals.map.jinja +++ b/salt/vars/globals.map.jinja @@ -21,7 +21,7 @@ 'influxdb_host': INIT.PILLAR.global.influxdb_host, 'manager_ip': INIT.PILLAR.global.managerip, 'md_engine': INIT.PILLAR.global.mdengine, - 'pcap_engine': GLOBALMERGED.pcapengine + 'pcap_engine': GLOBALMERGED.pcapengine, 'pipeline': INIT.PILLAR.global.pipeline, 'so_version': INIT.PILLAR.global.soversion, 'so_docker_gateway': DOCKER.gateway, From 0522dc180a753ddb5886c9b370df6fc6662eef10 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 13:39:35 -0500 Subject: [PATCH 10/19] map pcap dir to container. enable pcap-log in map --- salt/suricata/enabled.sls | 3 +++ salt/suricata/map.jinja | 5 +++++ salt/suricata/pcap.sls | 4 ++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index cf871906b..8b2776b39 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -35,6 +35,9 @@ so-suricata: - /nsm/suricata/:/nsm/:rw - /nsm/suricata/extracted:/var/log/suricata//filestore:rw - /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro + {% if GLOBALS.pcap_engine == "SURICATA" %} + - /nsm/suripcap/:/nsm/pcap:rw + {% endif %} {% if DOCKER.containers['so-suricata'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-suricata'].custom_bind_mounts %} - {{ BIND }} diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 01d019de8..5f6e913f5 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -60,6 +60,11 @@ {% do SURICATAMERGED.config.outputs['file-store'].update({'enabled':suricata_mdengine.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %} {% endif %} +{# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #} +{% if GLOBALS.pcap_engine == "SURICATA" %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %} +{% endif %} + {# outputs is a list but we convert to dict in defaults to work with ui #} {# below they are converted back to lists #} {% load_yaml as outputs %} diff --git a/salt/suricata/pcap.sls b/salt/suricata/pcap.sls index 7a00d8d6a..a3cbafa0a 100644 --- a/salt/suricata/pcap.sls +++ b/salt/suricata/pcap.sls @@ -9,8 +9,8 @@ suripcapdir: - mode: 755 - makedirs: True - -{% for i in range(1, SURICATAMERGED.config['af-packet'].threads) + 1) %} +{# there should only be 1 interface in af-packet so we can just reference the first list item #} +{% for i in range(1, SURICATAMERGED.config['af-packet'][0].threads + 1) %} suripcapthread{{i}}dir: file.directory: From 8ed66ea468b5eea0d3230db0e23ea34fd3267762 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 15:22:32 -0500 Subject: [PATCH 11/19] disable stenographer if suricata is pcap engine --- salt/pcap/config.map.jinja | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/pcap/config.map.jinja b/salt/pcap/config.map.jinja index 7ed500f25..e6d9f8bda 100644 --- a/salt/pcap/config.map.jinja +++ b/salt/pcap/config.map.jinja @@ -2,6 +2,12 @@ or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at https://securityonion.net/license; you may not use this file except in compliance with the Elastic License 2.0. #} - + +{% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'pcap/defaults.yaml' as PCAPDEFAULTS %} {% set PCAPMERGED = salt['pillar.get']('pcap', PCAPDEFAULTS.pcap, merge=True) %} + +{# disable stenographer if the pcap engine is set to SURICATA #} +{% if GLOBALS.pcap_engine == "SURICATA" %} +{% do PCAPMERGED.update({'enabled': False}) %} +{% endif %} From f32cb1f1153d691394f58295770717b781929ed9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 15:48:10 -0500 Subject: [PATCH 12/19] fix find to work with steno and suri pcap --- salt/telegraf/scripts/oldpcap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index b68e71539..438ce912c 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -15,7 +15,7 @@ PCAPLOC=/host/nsm/pcap if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then # Get the data - OLDPCAP=$(find $PCAPLOC -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) + OLDPCAP=$(find $PCAPLOC -type f -exec stat -c'%n %Z' {} + | sort | grep -v "/\." | head -n 1 | awk {'print $2'}) DATE=$(date +%s) AGE=$(($DATE - $OLDPCAP)) From 8b503e2ffa722977841947590195b1aae1a90663 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 15:58:11 -0500 Subject: [PATCH 13/19] telegraf dont run stenoloss script if suricata is pcap engine --- salt/telegraf/map.jinja | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/telegraf/map.jinja b/salt/telegraf/map.jinja index e6d3460d6..b56c8a64d 100644 --- a/salt/telegraf/map.jinja +++ b/salt/telegraf/map.jinja @@ -14,4 +14,11 @@ {% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %} {% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %} {% endif %} + +{% from 'pcap/config.map.jinja' import PCAPMERGED %} +{# PCAPMERGED.enabled is set false in soc ui or if suricata is the pcap engine #} +{% if not PCAPMERGED.enabled %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('stenoloss.sh') %} +{% endif %} + {% endif %} From 8a25748e3309fea6b8f5f01537e4c8f9bdafc65f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 16:06:24 -0500 Subject: [PATCH 14/19] grammar --- salt/global/soc_global.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index fc1c09b1c..d707fb1cc 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -10,12 +10,12 @@ global: regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ regexFailureMessage: You must enter a valid IP address or CIDR. mdengine: - description: What engine to use for meta data generation. Options are ZEEK and SURICATA. + description: Which engine to use for meta data generation. Options are ZEEK and SURICATA. regex: ^(ZEEK|SURICATA)$ regexFailureMessage: You must enter either ZEEK or SURICATA. global: True pcapengine: - description: What engine to use for generating pcap. Options are STENO and SURICATA. + description: Which engine to use for generating pcap. Options are STENO and SURICATA. regex: ^(STENO|SURICATA)$ regexFailureMessage: You must enter either STENO or SURICATA. global: True From 4be1214bab11f57286f042ce7dbebd76bcbb8259 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 16:53:57 -0500 Subject: [PATCH 15/19] pcap engine logic for sensoroni --- salt/sensoroni/files/sensoroni.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index f813dad2f..97c91f0b3 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -1,6 +1,7 @@ {%- from 'vars/globals.map.jinja' import GLOBALS %} {%- from 'sensoroni/map.jinja' import SENSORONIMERGED %} {%- from 'pcap/config.map.jinja' import PCAPMERGED %} +{%- from 'suricata/map.jinja' import SURICATAMERGED %} { "logFilename": "/opt/sensoroni/logs/sensoroni.log", "logLevel":"info", @@ -23,22 +24,22 @@ "importer": {}, "statickeyauth": { "apiKey": "{{ GLOBALS.sensoroni_key }}" +{#- if PCAPMERGED.enabled is true then we know that steno is the pcap engine #} +{#- if it is false, then user has steno disabled in ui or has selected suricata for pcap engine #} {%- if PCAPMERGED.enabled %} -{%- if GLOBALS.pcap_engine == "STENO" %} }, "stenoquery": { "executablePath": "/opt/sensoroni/scripts/stenoquery.sh", "pcapInputPath": "/nsm/pcap", "pcapOutputPath": "/nsm/pcapout" } -{%- elif GLOBALS.pcap_engine == "SURICATA" %} +{%- elif GLOBALS.pcap_engine == "SURICATA" and SURICATAMERGED.enabled %} }, "suriquery": { "executablePath": "/opt/sensoroni/scripts/suriquery.sh", "pcapInputPath": "/nsm/suripcap", "pcapOutputPath": "/nsm/pcapout" } -{%- endif %} {%- else %} } {%- endif %} From 00289c201ee39e5bba2836f30106dbe7957acf61 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Jan 2024 08:58:57 -0500 Subject: [PATCH 16/19] fix pcap paths --- salt/sensoroni/enabled.sls | 3 +++ salt/suricata/defaults.yaml | 2 +- salt/suricata/enabled.sls | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/sensoroni/enabled.sls b/salt/sensoroni/enabled.sls index 2111e8f1b..07b4df10a 100644 --- a/salt/sensoroni/enabled.sls +++ b/salt/sensoroni/enabled.sls @@ -23,6 +23,9 @@ so-sensoroni: - /opt/so/conf/sensoroni/sensoroni.json:/opt/sensoroni/sensoroni.json:ro - /opt/so/conf/sensoroni/analyzers:/opt/sensoroni/analyzers:rw - /opt/so/log/sensoroni:/opt/sensoroni/logs:rw + {% if GLOBALS.pcap_engine == "SURICATA" %} + - /nsm/suripcap/:/nsm/suripcap:rw + {% endif %} {% if DOCKER.containers['so-sensoroni'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-sensoroni'].custom_bind_mounts %} - {{ BIND }} diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index 4961ae50a..eb2c181e3 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -137,7 +137,7 @@ suricata: max-files: 10 use-stream-depth: "no" conditional: "all" - dir: "/nsm/pcap" + dir: "/nsm/suripcap" alert-debug: enabled: "no" alert-prelude: diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index 8b2776b39..fa1ebafef 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -36,7 +36,7 @@ so-suricata: - /nsm/suricata/extracted:/var/log/suricata//filestore:rw - /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro {% if GLOBALS.pcap_engine == "SURICATA" %} - - /nsm/suripcap/:/nsm/pcap:rw + - /nsm/suripcap/:/nsm/suripcap:rw {% endif %} {% if DOCKER.containers['so-suricata'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-suricata'].custom_bind_mounts %} From 0d01d09d2e8805287cfd061038b1df64ff1348c5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Jan 2024 09:15:35 -0500 Subject: [PATCH 17/19] fix pcap paths --- salt/sensoroni/enabled.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/sensoroni/enabled.sls b/salt/sensoroni/enabled.sls index 07b4df10a..6dc3df2bd 100644 --- a/salt/sensoroni/enabled.sls +++ b/salt/sensoroni/enabled.sls @@ -25,6 +25,7 @@ so-sensoroni: - /opt/so/log/sensoroni:/opt/sensoroni/logs:rw {% if GLOBALS.pcap_engine == "SURICATA" %} - /nsm/suripcap/:/nsm/suripcap:rw + - /nsm/suripcaptmp:/nsm/suripcaptmp:rw {% endif %} {% if DOCKER.containers['so-sensoroni'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-sensoroni'].custom_bind_mounts %} From 018e099111e86774e51126b42d22253b73ea7495 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 4 Mar 2024 14:53:15 -0500 Subject: [PATCH 18/19] Modify setup --- salt/manager/tools/sbin/so-minion | 39 +++++++++++++++++++------------ 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 4995e1c9d..4a6e5b7c7 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -80,28 +80,27 @@ function getinstallinfo() { } function pcapspace() { - - local NSMSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') - local ROOTSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') - - if [[ "$NSMSIZE" == "null" ]]; then - # Looks like there is no dedicated nsm partition. Using root - local SPACESIZE=$ROOTSIZE + if [[ "$OPERATION" == "setup" ]]; then + local SPACESIZE=$(df -h /nsm | tail -1 | awk '{print $2}') else - local SPACESIZE=$NSMSIZE + + local NSMSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') + local ROOTSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') + + if [[ "$NSMSIZE" == "null" ]]; then + # Looks like there is no dedicated nsm partition. Using root + local SPACESIZE=$ROOTSIZE + else + local SPACESIZE=$NSMSIZE + fi fi local s=$(( $SPACESIZE / 1000000 )) local s1=$(( $s / 2 )) local s2=$(( $s1 / $lb_procs )) - printf '%s\n'\ - "suricata:"\ - " config:"\ - " output:"\ - " pcap-log:"\ - " max-files: $s" >> $PILLARFILE - + MAXPCAPFILES=$s2 + } function testMinion() { @@ -272,6 +271,11 @@ function add_sensor_to_minion() { echo " config:" >> $PILLARFILE echo " af-packet:" >> $PILLARFILE echo " threads: '$CORECOUNT'" >> $PILLARFILE + if [[ $is_pcaplimit ]]; then + echo " output:" >> $PILLARFILE + echo " pcap-log:" >> $PILLARFILE + echo " max-files: '$MAXPCAPFILES'" >> $PILLARFILE + fi echo "pcap:" >> $PILLARFILE echo " enabled: True" >> $PILLARFILE if [[ $is_pcaplimit ]]; then @@ -448,6 +452,7 @@ function updateMine() { function createEVAL() { is_pcaplimit=true + pcapspace add_elasticsearch_to_minion add_sensor_to_minion add_strelka_to_minion @@ -468,6 +473,7 @@ function createEVAL() { function createSTANDALONE() { is_pcaplimit=true + pcapspace add_elasticsearch_to_minion add_logstash_to_minion add_sensor_to_minion @@ -557,6 +563,7 @@ function createIDH() { function createHEAVYNODE() { is_pcaplimit=true + pcapspace add_elasticsearch_to_minion add_elastic_agent_to_minion add_logstash_to_minion @@ -567,6 +574,8 @@ function createHEAVYNODE() { } function createSENSOR() { + is_pcaplimit=true + pcapspace add_sensor_to_minion add_strelka_to_minion add_telegraf_to_minion From fe238755e981fdc3316c48f03c23c9a6a29613ca Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 4 Mar 2024 16:52:51 -0500 Subject: [PATCH 19/19] Fix df --- salt/manager/tools/sbin/so-minion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 4a6e5b7c7..d696e14c6 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -81,7 +81,7 @@ function getinstallinfo() { function pcapspace() { if [[ "$OPERATION" == "setup" ]]; then - local SPACESIZE=$(df -h /nsm | tail -1 | awk '{print $2}') + local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n) else local NSMSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ')