From 762a3bea177a021dc97a1dcf9c1771d9c8bcc359 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 25 Jan 2024 09:59:26 -0500 Subject: [PATCH 001/140] Defaults and Annotations --- salt/suricata/defaults.yaml | 10 +++++++ salt/suricata/soc_suricata.yaml | 50 +++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index e9e39d40a..4961ae50a 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -128,6 +128,16 @@ suricata: enabled: "no" pcap-log: enabled: "no" + compression: "none" + lz4-checksum: "no" + lz4-level: 8 + filename: "%n/so-pcap.%t" + limit: "1000mb" + mode: "multi" + max-files: 10 + use-stream-depth: "no" + conditional: "all" + dir: "/nsm/pcap" alert-debug: enabled: "no" alert-prelude: diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 30f277c0a..58a2273b9 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -153,6 +153,53 @@ suricata: header: description: Header name where the actual IP address will be reported. helpLink: suricata.html + pcap-log: + enabled: + description: Enable Suricata to collect PCAP. + helpLink: suricata.html + compression: + description: Enable compression of Suricata PCAP. Currently unsupported + advanced: True + readonly: True + helpLink: suricata.html + lz4-checksum: + description: Enable PCAP lz4 checksum. Currently unsupported + advanced: True + readonly: True + helpLink: suricata.html + lz4-level: + description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported + advanced: True + readonly: True + helpLink: suricata.html + filename: + description: Filename output for Suricata PCAP. + advanced: True + readonly: True + helpLink: suricata.html + limit: + description: File size limit per thread. To determine max PCAP size multiple threads x max-files x limit. + helpLink: suricata.html + mode: + description: Suricata PCAP mode. Currenlty only multi is supported. + advanced: True + readonly: True + helpLink: suricata.html + max-files: + description: Max PCAP files per thread. To determine max PCAP size multiple threads x max-files x limit. + helpLink: suricata.html + use-stream-depth: + description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. + advanced: True + helpLink: suricata.html + conditional: + description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules. + helpLink: suricata.html + dir: + description: Parent directory to store PCAP. + advanced: True + readonly: True + helpLink: suricata.html asn1-max-frames: description: Maximum nuber of asn1 frames to decode. helpLink: suricata.html @@ -209,6 +256,9 @@ suricata: memcap: description: Can be specified in kb,mb,gb. helpLink: suricata.html + depth: + description: Controls how far into a stream that reassembly is done. + helpLink: suricata.html host: hash-size: description: Hash size in bytes. From 1a2245a1ed188be34b2ffbcb6304567c9c5abdee Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 13:44:53 -0500 Subject: [PATCH 002/140] Add so-minion modifications --- salt/manager/tools/sbin/so-minion | 25 +++++++++++++++++++++++++ salt/suricata/soc_suricata.yaml | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index d5225cc82..12349b680 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -79,6 +79,30 @@ function getinstallinfo() { source <(echo $INSTALLVARS) } +function pcapspace() { + + local NSMSIZE=$(salt \* disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') + local ROOTSIZE=$(salt \* disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') + + if [[ "$NSMSIZE" == "null" ]]; then + # Looks like there is no dedicated nsm partition. Using root + local SPACESIZE=$ROOTSIZE + else + local SPACESIZE=$NSMSIZE + fi + + local s=$(( $SPACESIZE / 1000000 )) + local s1=$(( $s / 2 )) + local s2=$(( $s1 / $lb_procs )) + + printf '%s\n'\ + "suricata:"\ + " config:"\ + " output:"\ + " pcap-log: $s" >> $PILLARFILE + +} + function testMinion() { # Always run on the host, since this is going to be the manager of a distributed grid, or an eval/standalone. # Distributed managers must run this in order for the sensor nodes to have access to the so-tcpreplay image. @@ -252,6 +276,7 @@ function add_sensor_to_minion() { if [[ $is_pcaplimit ]]; then echo " config:" >> $PILLARFILE echo " diskfreepercentage: 60" >> $PILLARFILE + pcapspace fi echo " " >> $PILLARFILE } diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 58a2273b9..5dddd7442 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -181,7 +181,7 @@ suricata: description: File size limit per thread. To determine max PCAP size multiple threads x max-files x limit. helpLink: suricata.html mode: - description: Suricata PCAP mode. Currenlty only multi is supported. + description: Suricata PCAP mode. Currently only multi is supported. advanced: True readonly: True helpLink: suricata.html From 5b05aec96aaa9cbf73294bce24fed2d59f5e4be4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 14:56:51 -0500 Subject: [PATCH 003/140] Target sspecific minion --- salt/manager/tools/sbin/so-minion | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 12349b680..877796620 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -81,8 +81,8 @@ function getinstallinfo() { function pcapspace() { - local NSMSIZE=$(salt \* disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') - local ROOTSIZE=$(salt \* disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') + local NSMSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') + local ROOTSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') if [[ "$NSMSIZE" == "null" ]]; then # Looks like there is no dedicated nsm partition. Using root From 0c969312e2ee61eed8e6c0d571e44dc7ee7cdcec Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 15:22:20 -0500 Subject: [PATCH 004/140] Add Globals --- salt/global/defaults.yaml | 2 ++ salt/global/soc_global.yaml | 5 +++++ salt/manager/tools/sbin/so-minion | 3 ++- salt/sensoroni/files/sensoroni.json | 11 ++++++++++- salt/suricata/soc_suricata.yaml | 4 +++- 5 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 salt/global/defaults.yaml diff --git a/salt/global/defaults.yaml b/salt/global/defaults.yaml new file mode 100644 index 000000000..bd7244a58 --- /dev/null +++ b/salt/global/defaults.yaml @@ -0,0 +1,2 @@ +global: + pcapengine: STENO \ No newline at end of file diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index 14d637d50..fc1c09b1c 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -14,6 +14,11 @@ global: regex: ^(ZEEK|SURICATA)$ regexFailureMessage: You must enter either ZEEK or SURICATA. global: True + pcapengine: + description: What engine to use for generating pcap. Options are STENO and SURICATA. + regex: ^(STENO|SURICATA)$ + regexFailureMessage: You must enter either STENO or SURICATA. + global: True ids: description: Which IDS engine to use. Currently only Suricata is supported. global: True diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 877796620..4995e1c9d 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -99,7 +99,8 @@ function pcapspace() { "suricata:"\ " config:"\ " output:"\ - " pcap-log: $s" >> $PILLARFILE + " pcap-log:"\ + " max-files: $s" >> $PILLARFILE } diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 59ce500e3..c5608ba56 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -23,13 +23,22 @@ "importer": {}, "statickeyauth": { "apiKey": "{{ GLOBALS.sensoroni_key }}" -{%- if PCAPMERGED.enabled %} +{%- if PCAPMERGED.enabled %} +{%- if PCAPENGINE.steno %} }, "stenoquery": { "executablePath": "/opt/sensoroni/scripts/stenoquery.sh", "pcapInputPath": "/nsm/pcap", "pcapOutputPath": "/nsm/pcapout" } +{%- elif PCAPENGINE.suri %} + }, + "suriquery": { + "executablePath": "/opt/sensoroni/scripts/suriquery.sh", + "pcapInputPath": "/nsm/suripcap", + "pcapOutputPath": "/nsm/pcapout" + } +{%- endif %} {%- else %} } {%- endif %} diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 5dddd7442..7153eb9a1 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -155,8 +155,10 @@ suricata: helpLink: suricata.html pcap-log: enabled: - description: Enable Suricata to collect PCAP. + description: This value is ignored by SO. pcapengine in globals takes predidence. + readonly: True helpLink: suricata.html + advanced: True compression: description: Enable compression of Suricata PCAP. Currently unsupported advanced: True From 88c01a22d6454b318c0c94a1718c93fb69b34500 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 15:27:28 -0500 Subject: [PATCH 005/140] Add annotation logic --- salt/suricata/soc_suricata.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 7153eb9a1..52352d043 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -44,6 +44,7 @@ suricata: set-cpu-affinity: description: Bind(yes) or unbind(no) management and worker threads to a core or range of cores. regex: ^(yes|no)$ + regexFailureMessage: You must enter either yes or no. helpLink: suricata.html cpu-affinity: management-cpu-set: @@ -155,7 +156,7 @@ suricata: helpLink: suricata.html pcap-log: enabled: - description: This value is ignored by SO. pcapengine in globals takes predidence. + description: This value is ignored by SO. pcapengine in globals takes precidence. readonly: True helpLink: suricata.html advanced: True @@ -193,9 +194,13 @@ suricata: use-stream-depth: description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. advanced: True + regex: ^(yes|no)$ + regexFailureMessage: You must enter either yes or no. helpLink: suricata.html conditional: description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules. + regex: ^(all|alert|tag)$ + regexFailureMessage: You must enter either all, alert or tag. helpLink: suricata.html dir: description: Parent directory to store PCAP. From ab551a747ddafe1fd0b602e3f84d8130e9ffe5bc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 15:44:57 -0500 Subject: [PATCH 006/140] Threads placeholder logic --- salt/suricata/enabled.sls | 3 +++ salt/suricata/pcap.sls | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 salt/suricata/pcap.sls diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index ce309e41a..6dce49c8c 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -12,6 +12,9 @@ include: - suricata.config - suricata.sostatus + if blah + - suricata.pcap + endif so-suricata: docker_container.running: diff --git a/salt/suricata/pcap.sls b/salt/suricata/pcap.sls new file mode 100644 index 000000000..f677532f0 --- /dev/null +++ b/salt/suricata/pcap.sls @@ -0,0 +1,25 @@ +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% import_yaml 'suricata/defaults.yaml' as SURICATADEFAULTS %} +{% set SURICATAMERGED = salt['pillar.get']('suricata', SURICATADEFAULTS.suricata, merge=True) %} + +suripcapdir: + file.directory: + - name: /nsm/suripcap + - user: 940 + - group: 939 + - mode: 755 + - makedirs: True + +{{ SURICATAMERGED.config['af-packet'].threads }} + +for thread in afp.threads + +suripcapthreaddir: + file.directory: + - name: /nsm/suripcap/{{thread}} + - user: 940 + - group: 939 + - mode: 755 + - makedirs: True + +endfor \ No newline at end of file From 88d2ddba8bddeeac28fbadf12c826c04a4a61e82 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 15:53:54 -0500 Subject: [PATCH 007/140] add placeholder for telegraf --- salt/telegraf/scripts/oldpcap.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index bb1be457f..d3f4b9a93 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -5,13 +5,18 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +{%- if pcap is steno +PCAPLOC=/host/nsm/pcap +{%- else %} +PCAPLOC=/host/nsm/suripcap +{%- endif %} # if this script isn't already running if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then # Get the data - OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) + OLDPCAP=$(find $PCAPLOC -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) DATE=$(date +%s) AGE=$(($DATE - $OLDPCAP)) From d118ff4728454cb08cd020234b6d858c5edc3656 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 29 Jan 2024 16:54:08 -0500 Subject: [PATCH 008/140] add GLOBALS.pcap_engine --- salt/global/map.jinja | 2 ++ salt/sensoroni/files/sensoroni.json | 6 +++--- salt/suricata/enabled.sls | 4 ++-- salt/suricata/pcap.sls | 13 +++++-------- salt/telegraf/config.sls | 2 ++ salt/telegraf/scripts/oldpcap.sh | 7 +++---- salt/vars/globals.map.jinja | 2 ++ 7 files changed, 19 insertions(+), 17 deletions(-) create mode 100644 salt/global/map.jinja diff --git a/salt/global/map.jinja b/salt/global/map.jinja new file mode 100644 index 000000000..54abb8c79 --- /dev/null +++ b/salt/global/map.jinja @@ -0,0 +1,2 @@ +{% import_yaml 'global/defaults.yaml' as GLOBALDEFAULTS %} +{% set GLOBALMERGED = salt['pillar.get']('global', GLOBALDEFAULTS.global, merge=True) %} diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index c5608ba56..f813dad2f 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -24,21 +24,21 @@ "statickeyauth": { "apiKey": "{{ GLOBALS.sensoroni_key }}" {%- if PCAPMERGED.enabled %} -{%- if PCAPENGINE.steno %} +{%- if GLOBALS.pcap_engine == "STENO" %} }, "stenoquery": { "executablePath": "/opt/sensoroni/scripts/stenoquery.sh", "pcapInputPath": "/nsm/pcap", "pcapOutputPath": "/nsm/pcapout" } -{%- elif PCAPENGINE.suri %} +{%- elif GLOBALS.pcap_engine == "SURICATA" %} }, "suriquery": { "executablePath": "/opt/sensoroni/scripts/suriquery.sh", "pcapInputPath": "/nsm/suripcap", "pcapOutputPath": "/nsm/pcapout" } -{%- endif %} +{%- endif %} {%- else %} } {%- endif %} diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index 6dce49c8c..cf871906b 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -12,9 +12,9 @@ include: - suricata.config - suricata.sostatus - if blah +{% if GLOBALS.pcap_engine == "SURICATA" %} - suricata.pcap - endif +{% endif %} so-suricata: docker_container.running: diff --git a/salt/suricata/pcap.sls b/salt/suricata/pcap.sls index f677532f0..7a00d8d6a 100644 --- a/salt/suricata/pcap.sls +++ b/salt/suricata/pcap.sls @@ -1,6 +1,5 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} -{% import_yaml 'suricata/defaults.yaml' as SURICATADEFAULTS %} -{% set SURICATAMERGED = salt['pillar.get']('suricata', SURICATADEFAULTS.suricata, merge=True) %} +{% from 'suricata/map.jinja' import SURICATAMERGED %} suripcapdir: file.directory: @@ -10,16 +9,14 @@ suripcapdir: - mode: 755 - makedirs: True -{{ SURICATAMERGED.config['af-packet'].threads }} -for thread in afp.threads +{% for i in range(1, SURICATAMERGED.config['af-packet'].threads) + 1) %} -suripcapthreaddir: +suripcapthread{{i}}dir: file.directory: - - name: /nsm/suripcap/{{thread}} + - name: /nsm/suripcap/{{i}} - user: 940 - group: 939 - mode: 755 - - makedirs: True -endfor \ No newline at end of file +{% endfor %} diff --git a/salt/telegraf/config.sls b/salt/telegraf/config.sls index 0711260b5..a35be55f5 100644 --- a/salt/telegraf/config.sls +++ b/salt/telegraf/config.sls @@ -41,6 +41,8 @@ tgraf_sync_script_{{script}}: - mode: 770 - template: jinja - source: salt://telegraf/scripts/{{script}} + - defaults: + GLOBALS: {{ GLOBALS }} {% endfor %} telegraf_sbin: diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index d3f4b9a93..b68e71539 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -5,13 +5,12 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- if pcap is steno -PCAPLOC=/host/nsm/pcap -{%- else %} +{%- if GLOBALS.pcap_engine == "SURICATA" %} PCAPLOC=/host/nsm/suripcap +{%- else %} +PCAPLOC=/host/nsm/pcap {%- endif %} - # if this script isn't already running if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then diff --git a/salt/vars/globals.map.jinja b/salt/vars/globals.map.jinja index 3265cde18..990aeb39b 100644 --- a/salt/vars/globals.map.jinja +++ b/salt/vars/globals.map.jinja @@ -1,5 +1,6 @@ {% import 'vars/init.map.jinja' as INIT %} {% from 'docker/docker.map.jinja' import DOCKER %} +{% from 'global/map.jinja' import GLOBALMERGED %} {% from 'vars/' ~ INIT.GRAINS.role.split('-')[1] ~ '.map.jinja' import ROLE_GLOBALS %} {# role is so-role so we have to split off the 'so' #} @@ -20,6 +21,7 @@ 'influxdb_host': INIT.PILLAR.global.influxdb_host, 'manager_ip': INIT.PILLAR.global.managerip, 'md_engine': INIT.PILLAR.global.mdengine, + 'pcap_engine': GLOBALMERGED.pcapengine 'pipeline': INIT.PILLAR.global.pipeline, 'so_version': INIT.PILLAR.global.soversion, 'so_docker_gateway': DOCKER.gateway, From 37dcb84a09d836ec1b772fe25c42051e7bfdf797 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 10:50:01 -0500 Subject: [PATCH 009/140] add missing comma --- salt/vars/globals.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/vars/globals.map.jinja b/salt/vars/globals.map.jinja index 990aeb39b..624173217 100644 --- a/salt/vars/globals.map.jinja +++ b/salt/vars/globals.map.jinja @@ -21,7 +21,7 @@ 'influxdb_host': INIT.PILLAR.global.influxdb_host, 'manager_ip': INIT.PILLAR.global.managerip, 'md_engine': INIT.PILLAR.global.mdengine, - 'pcap_engine': GLOBALMERGED.pcapengine + 'pcap_engine': GLOBALMERGED.pcapengine, 'pipeline': INIT.PILLAR.global.pipeline, 'so_version': INIT.PILLAR.global.soversion, 'so_docker_gateway': DOCKER.gateway, From 0522dc180a753ddb5886c9b370df6fc6662eef10 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 13:39:35 -0500 Subject: [PATCH 010/140] map pcap dir to container. enable pcap-log in map --- salt/suricata/enabled.sls | 3 +++ salt/suricata/map.jinja | 5 +++++ salt/suricata/pcap.sls | 4 ++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index cf871906b..8b2776b39 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -35,6 +35,9 @@ so-suricata: - /nsm/suricata/:/nsm/:rw - /nsm/suricata/extracted:/var/log/suricata//filestore:rw - /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro + {% if GLOBALS.pcap_engine == "SURICATA" %} + - /nsm/suripcap/:/nsm/pcap:rw + {% endif %} {% if DOCKER.containers['so-suricata'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-suricata'].custom_bind_mounts %} - {{ BIND }} diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 01d019de8..5f6e913f5 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -60,6 +60,11 @@ {% do SURICATAMERGED.config.outputs['file-store'].update({'enabled':suricata_mdengine.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %} {% endif %} +{# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #} +{% if GLOBALS.pcap_engine == "SURICATA" %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %} +{% endif %} + {# outputs is a list but we convert to dict in defaults to work with ui #} {# below they are converted back to lists #} {% load_yaml as outputs %} diff --git a/salt/suricata/pcap.sls b/salt/suricata/pcap.sls index 7a00d8d6a..a3cbafa0a 100644 --- a/salt/suricata/pcap.sls +++ b/salt/suricata/pcap.sls @@ -9,8 +9,8 @@ suripcapdir: - mode: 755 - makedirs: True - -{% for i in range(1, SURICATAMERGED.config['af-packet'].threads) + 1) %} +{# there should only be 1 interface in af-packet so we can just reference the first list item #} +{% for i in range(1, SURICATAMERGED.config['af-packet'][0].threads + 1) %} suripcapthread{{i}}dir: file.directory: From 8ed66ea468b5eea0d3230db0e23ea34fd3267762 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 15:22:32 -0500 Subject: [PATCH 011/140] disable stenographer if suricata is pcap engine --- salt/pcap/config.map.jinja | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/pcap/config.map.jinja b/salt/pcap/config.map.jinja index 7ed500f25..e6d9f8bda 100644 --- a/salt/pcap/config.map.jinja +++ b/salt/pcap/config.map.jinja @@ -2,6 +2,12 @@ or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at https://securityonion.net/license; you may not use this file except in compliance with the Elastic License 2.0. #} - + +{% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'pcap/defaults.yaml' as PCAPDEFAULTS %} {% set PCAPMERGED = salt['pillar.get']('pcap', PCAPDEFAULTS.pcap, merge=True) %} + +{# disable stenographer if the pcap engine is set to SURICATA #} +{% if GLOBALS.pcap_engine == "SURICATA" %} +{% do PCAPMERGED.update({'enabled': False}) %} +{% endif %} From f32cb1f1153d691394f58295770717b781929ed9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 15:48:10 -0500 Subject: [PATCH 012/140] fix find to work with steno and suri pcap --- salt/telegraf/scripts/oldpcap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index b68e71539..438ce912c 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -15,7 +15,7 @@ PCAPLOC=/host/nsm/pcap if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then # Get the data - OLDPCAP=$(find $PCAPLOC -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) + OLDPCAP=$(find $PCAPLOC -type f -exec stat -c'%n %Z' {} + | sort | grep -v "/\." | head -n 1 | awk {'print $2'}) DATE=$(date +%s) AGE=$(($DATE - $OLDPCAP)) From 8b503e2ffa722977841947590195b1aae1a90663 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 15:58:11 -0500 Subject: [PATCH 013/140] telegraf dont run stenoloss script if suricata is pcap engine --- salt/telegraf/map.jinja | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/telegraf/map.jinja b/salt/telegraf/map.jinja index e6d3460d6..b56c8a64d 100644 --- a/salt/telegraf/map.jinja +++ b/salt/telegraf/map.jinja @@ -14,4 +14,11 @@ {% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %} {% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %} {% endif %} + +{% from 'pcap/config.map.jinja' import PCAPMERGED %} +{# PCAPMERGED.enabled is set false in soc ui or if suricata is the pcap engine #} +{% if not PCAPMERGED.enabled %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('stenoloss.sh') %} +{% endif %} + {% endif %} From 8a25748e3309fea6b8f5f01537e4c8f9bdafc65f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 16:06:24 -0500 Subject: [PATCH 014/140] grammar --- salt/global/soc_global.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index fc1c09b1c..d707fb1cc 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -10,12 +10,12 @@ global: regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ regexFailureMessage: You must enter a valid IP address or CIDR. mdengine: - description: What engine to use for meta data generation. Options are ZEEK and SURICATA. + description: Which engine to use for meta data generation. Options are ZEEK and SURICATA. regex: ^(ZEEK|SURICATA)$ regexFailureMessage: You must enter either ZEEK or SURICATA. global: True pcapengine: - description: What engine to use for generating pcap. Options are STENO and SURICATA. + description: Which engine to use for generating pcap. Options are STENO and SURICATA. regex: ^(STENO|SURICATA)$ regexFailureMessage: You must enter either STENO or SURICATA. global: True From 0fa4d92f8ff8beb7d8e597bf357ab3a578c04c51 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 19 Oct 2023 15:49:56 -0600 Subject: [PATCH 015/140] socsigmarepo Need write permissions on the /opt/so/rules dir so I can clone the sigma repo there. --- salt/soc/config.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 902d82ec7..95135566b 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -114,6 +114,13 @@ socuploaddir: - group: 939 - makedirs: True +socsigmarepo: + file.directory: + - name: /opt/so/rules + - user: 939 + - group: 939 + - mode: 775 + {% else %} {{sls}}_state_not_allowed: From 4be1214bab11f57286f042ce7dbebd76bcbb8259 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 16:53:57 -0500 Subject: [PATCH 016/140] pcap engine logic for sensoroni --- salt/sensoroni/files/sensoroni.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index f813dad2f..97c91f0b3 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -1,6 +1,7 @@ {%- from 'vars/globals.map.jinja' import GLOBALS %} {%- from 'sensoroni/map.jinja' import SENSORONIMERGED %} {%- from 'pcap/config.map.jinja' import PCAPMERGED %} +{%- from 'suricata/map.jinja' import SURICATAMERGED %} { "logFilename": "/opt/sensoroni/logs/sensoroni.log", "logLevel":"info", @@ -23,22 +24,22 @@ "importer": {}, "statickeyauth": { "apiKey": "{{ GLOBALS.sensoroni_key }}" +{#- if PCAPMERGED.enabled is true then we know that steno is the pcap engine #} +{#- if it is false, then user has steno disabled in ui or has selected suricata for pcap engine #} {%- if PCAPMERGED.enabled %} -{%- if GLOBALS.pcap_engine == "STENO" %} }, "stenoquery": { "executablePath": "/opt/sensoroni/scripts/stenoquery.sh", "pcapInputPath": "/nsm/pcap", "pcapOutputPath": "/nsm/pcapout" } -{%- elif GLOBALS.pcap_engine == "SURICATA" %} +{%- elif GLOBALS.pcap_engine == "SURICATA" and SURICATAMERGED.enabled %} }, "suriquery": { "executablePath": "/opt/sensoroni/scripts/suriquery.sh", "pcapInputPath": "/nsm/suripcap", "pcapOutputPath": "/nsm/pcapout" } -{%- endif %} {%- else %} } {%- endif %} From 858166bcae280be03c5b416c912fd8c4a7ac8d61 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 30 Jan 2024 15:43:51 -0700 Subject: [PATCH 017/140] WIP: Detections Changes Removed some strelka/yara rules from salt. Removed yara scripts for downloading and updating rules. This will be managed by SOC. Added a new compile_yara.py script. Added the strelka repos folder. --- salt/manager/init.sls | 53 +------------------ .../manager/tools/sbin_jinja/so-yara-download | 51 ------------------ salt/manager/tools/sbin_jinja/so-yara-update | 41 -------------- salt/soc/files/bin/compile_yara.py | 14 +++++ salt/strelka/backend/config.sls | 10 ---- salt/strelka/config.sls | 9 +++- 6 files changed, 24 insertions(+), 154 deletions(-) delete mode 100644 salt/manager/tools/sbin_jinja/so-yara-download delete mode 100755 salt/manager/tools/sbin_jinja/so-yara-update create mode 100644 salt/soc/files/bin/compile_yara.py diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 23ef189b5..51590a6ec 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -1,5 +1,5 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -61,7 +61,7 @@ manager_sbin: - user: 939 - group: 939 - file_mode: 755 - - exclude_pat: + - exclude_pat: - "*_test.py" yara_update_scripts: @@ -103,55 +103,6 @@ rules_dir: - group: socore - makedirs: True -{% if STRELKAMERGED.rules.enabled %} - -strelkarepos: - file.managed: - - name: /opt/so/conf/strelka/repos.txt - - source: salt://strelka/rules/repos.txt.jinja - - template: jinja - - defaults: - STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} - - makedirs: True - -strelka-yara-update: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-update >> /opt/so/log/yarasync/yara-update.log 2>&1' - - identifier: strelka-yara-update - - hour: '7' - - minute: '1' - -strelka-yara-download: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-download >> /opt/so/log/yarasync/yara-download.log 2>&1' - - identifier: strelka-yara-download - - hour: '7' - - minute: '1' - -{% if not GLOBALS.airgap %} -update_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-update - - onchanges: - - file: yara_update_scripts - -download_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-download - - onchanges: - - file: yara_update_scripts -{% endif %} -{% endif %} {% else %} {{sls}}_state_not_allowed: diff --git a/salt/manager/tools/sbin_jinja/so-yara-download b/salt/manager/tools/sbin_jinja/so-yara-download deleted file mode 100644 index aa9576253..000000000 --- a/salt/manager/tools/sbin_jinja/so-yara-download +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -NOROOT=1 -. /usr/sbin/so-common - -{%- set proxy = salt['pillar.get']('manager:proxy') %} -{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} - -# Download the rules from the internet -{%- if proxy %} -export http_proxy={{ proxy }} -export https_proxy={{ proxy }} -export no_proxy="{{ noproxy }}" -{%- endif %} - -repos="/opt/so/conf/strelka/repos.txt" -output_dir=/nsm/rules/yara -gh_status=$(curl -s -o /dev/null -w "%{http_code}" https://github.com) -clone_dir="/tmp" -if [ "$gh_status" == "200" ] || [ "$gh_status" == "301" ]; then - - while IFS= read -r repo; do - if ! $(echo "$repo" | grep -qE '^#'); then - # Remove old repo if existing bc of previous error condition or unexpected disruption - repo_name=`echo $repo | awk -F '/' '{print $NF}'` - [ -d $output_dir/$repo_name ] && rm -rf $output_dir/$repo_name - - # Clone repo and make appropriate directories for rules - git clone $repo $clone_dir/$repo_name - echo "Analyzing rules from $clone_dir/$repo_name..." - mkdir -p $output_dir/$repo_name - # Ensure a copy of the license is available for the rules - [ -f $clone_dir/$repo_name/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name - - # Copy over rules - for i in $(find $clone_dir/$repo_name -name "*.yar*"); do - rule_name=$(echo $i | awk -F '/' '{print $NF}') - cp $i $output_dir/$repo_name - done - rm -rf $clone_dir/$repo_name - fi - done < $repos - - echo "Done!" - -/usr/sbin/so-yara-update - -else - echo "Server returned $gh_status status code." - echo "No connectivity to Github...exiting..." - exit 1 -fi diff --git a/salt/manager/tools/sbin_jinja/so-yara-update b/salt/manager/tools/sbin_jinja/so-yara-update deleted file mode 100755 index 07c940f47..000000000 --- a/salt/manager/tools/sbin_jinja/so-yara-update +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -NOROOT=1 -. /usr/sbin/so-common - -echo "Starting to check for yara rule updates at $(date)..." - -newcounter=0 -excludedcounter=0 -excluded_rules=({{ EXCLUDEDRULES | join(' ') }}) - -# Pull down the SO Rules -SORULEDIR=/nsm/rules/yara -OUTPUTDIR=/opt/so/saltstack/local/salt/strelka/rules - -mkdir -p $OUTPUTDIR -# remove all rules prior to copy so we can clear out old rules -rm -f $OUTPUTDIR/* - -for i in $(find $SORULEDIR -name "*.yar" -o -name "*.yara"); do - rule_name=$(echo $i | awk -F '/' '{print $NF}') - if [[ ! "${excluded_rules[*]}" =~ ${rule_name} ]]; then - echo "Adding rule: $rule_name..." - cp $i $OUTPUTDIR/$rule_name - ((newcounter++)) - else - echo "Excluding rule: $rule_name..." - ((excludedcounter++)) - fi -done - -if [ "$newcounter" -gt 0 ] || [ "$excludedcounter" -gt 0 ];then - echo "$newcounter rules added." - echo "$excludedcounter rule(s) excluded." -fi - -echo "Finished rule updates at $(date)..." diff --git a/salt/soc/files/bin/compile_yara.py b/salt/soc/files/bin/compile_yara.py new file mode 100644 index 000000000..43c8b1a09 --- /dev/null +++ b/salt/soc/files/bin/compile_yara.py @@ -0,0 +1,14 @@ +import os +import yara +import glob +import sys + +def compile_yara_rules(rules_dir: str) -> None: + compiled_rules_path: str = os.path.join(rules_dir, "rules.yar.compiled") + rule_files: list[str] = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True) + + if rule_files: + rules: yara.Rules = yara.compile(filepaths={os.path.basename(f): f for f in rule_files}) + rules.save(compiled_rules_path) + +compile_yara_rules(sys.argv[1]) diff --git a/salt/strelka/backend/config.sls b/salt/strelka/backend/config.sls index d51debb1b..b39e06ac8 100644 --- a/salt/strelka/backend/config.sls +++ b/salt/strelka/backend/config.sls @@ -50,16 +50,6 @@ backend_taste: - user: 939 - group: 939 -{% if STRELKAMERGED.rules.enabled %} -strelkarules: - file.recurse: - - name: /opt/so/conf/strelka/rules - - source: salt://strelka/rules - - user: 939 - - group: 939 - - clean: True -{% endif %} - {% else %} {{sls}}_state_not_allowed: diff --git a/salt/strelka/config.sls b/salt/strelka/config.sls index 1d0f75adf..929bef113 100644 --- a/salt/strelka/config.sls +++ b/salt/strelka/config.sls @@ -1,5 +1,5 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -21,6 +21,13 @@ strelkarulesdir: - group: 939 - makedirs: True +strelkareposdir: + file.directory: + - name: /opt/so/conf/strelka/repos + - user: 939 + - group: 939 + - makedirs: True + strelkadatadir: file.directory: - name: /nsm/strelka From 00289c201ee39e5bba2836f30106dbe7957acf61 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Jan 2024 08:58:57 -0500 Subject: [PATCH 018/140] fix pcap paths --- salt/sensoroni/enabled.sls | 3 +++ salt/suricata/defaults.yaml | 2 +- salt/suricata/enabled.sls | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/sensoroni/enabled.sls b/salt/sensoroni/enabled.sls index 2111e8f1b..07b4df10a 100644 --- a/salt/sensoroni/enabled.sls +++ b/salt/sensoroni/enabled.sls @@ -23,6 +23,9 @@ so-sensoroni: - /opt/so/conf/sensoroni/sensoroni.json:/opt/sensoroni/sensoroni.json:ro - /opt/so/conf/sensoroni/analyzers:/opt/sensoroni/analyzers:rw - /opt/so/log/sensoroni:/opt/sensoroni/logs:rw + {% if GLOBALS.pcap_engine == "SURICATA" %} + - /nsm/suripcap/:/nsm/suripcap:rw + {% endif %} {% if DOCKER.containers['so-sensoroni'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-sensoroni'].custom_bind_mounts %} - {{ BIND }} diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index 4961ae50a..eb2c181e3 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -137,7 +137,7 @@ suricata: max-files: 10 use-stream-depth: "no" conditional: "all" - dir: "/nsm/pcap" + dir: "/nsm/suripcap" alert-debug: enabled: "no" alert-prelude: diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index 8b2776b39..fa1ebafef 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -36,7 +36,7 @@ so-suricata: - /nsm/suricata/extracted:/var/log/suricata//filestore:rw - /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro {% if GLOBALS.pcap_engine == "SURICATA" %} - - /nsm/suripcap/:/nsm/pcap:rw + - /nsm/suripcap/:/nsm/suripcap:rw {% endif %} {% if DOCKER.containers['so-suricata'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-suricata'].custom_bind_mounts %} From 0d01d09d2e8805287cfd061038b1df64ff1348c5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Jan 2024 09:15:35 -0500 Subject: [PATCH 019/140] fix pcap paths --- salt/sensoroni/enabled.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/sensoroni/enabled.sls b/salt/sensoroni/enabled.sls index 07b4df10a..6dc3df2bd 100644 --- a/salt/sensoroni/enabled.sls +++ b/salt/sensoroni/enabled.sls @@ -25,6 +25,7 @@ so-sensoroni: - /opt/so/log/sensoroni:/opt/sensoroni/logs:rw {% if GLOBALS.pcap_engine == "SURICATA" %} - /nsm/suripcap/:/nsm/suripcap:rw + - /nsm/suripcaptmp:/nsm/suripcaptmp:rw {% endif %} {% if DOCKER.containers['so-sensoroni'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-sensoroni'].custom_bind_mounts %} From 585147d1de66d700849a23057f7af85c97421433 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 31 Jan 2024 10:39:47 -0700 Subject: [PATCH 020/140] Added so-detection mapping in elasticsearch --- salt/elasticsearch/defaults.yaml | 31 ++++- .../component/so/detection-mappings.json | 108 ++++++++++++++++++ .../component/so/detection-settings.json | 7 ++ 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 salt/elasticsearch/templates/component/so/detection-mappings.json create mode 100644 salt/elasticsearch/templates/component/so/detection-settings.json diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index ce1bfb08d..480467129 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -117,6 +117,35 @@ elasticsearch: sort: field: '@timestamp' order: desc + so-detection: + index_sorting: false + index_template: + composed_of: + - detection-mappings + - detection-settings + index_patterns: + - so-detection* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + mapping: + total_fields: + limit: 1500 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc so-common: close: 30 delete: 365 @@ -8909,7 +8938,7 @@ elasticsearch: actions: set_priority: priority: 50 - min_age: 30d + min_age: 30d so-logs-ti_otx_x_threat: index_sorting: false index_template: diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json new file mode 100644 index 000000000..df53308f2 --- /dev/null +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -0,0 +1,108 @@ +{ + "template": { + "mappings": { + "properties": { + "so_audit_doc_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "@timestamp": { + "type": "date" + }, + "so_kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_operation": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_detection": { + "properties": { + "publicId": { + "type": "text" + }, + "title": { + "type": "text" + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + }, + "author": { + "type": "text" + }, + "description": { + "type": "text" + }, + "content": { + "type": "text" + }, + "isEnabled": { + "type": "boolean" + }, + "isReporting": { + "type": "boolean" + }, + "isCommunity": { + "type": "boolean" + }, + "note": { + "type": "text" + }, + "engine": { + "ignore_above": 1024, + "type": "keyword" + }, + "overrides": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "isEnabled": { + "type": "boolean" + }, + "createdAt": { + "type": "date" + }, + "updatedAt": { + "type": "date" + }, + "regex": { + "type": "text" + }, + "value": { + "type": "text" + }, + "thresholdType": { + "ignore_above": 1024, + "type": "keyword" + }, + "track": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "text" + }, + "count": { + "type": "long" + }, + "seconds": { + "type": "long" + }, + "customFilter": { + "type": "text" + } + } + } + } + } + } + } + }, + "_meta": { + "ecs_version": "1.12.2" + } +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/so/detection-settings.json b/salt/elasticsearch/templates/component/so/detection-settings.json new file mode 100644 index 000000000..7b0947a4c --- /dev/null +++ b/salt/elasticsearch/templates/component/so/detection-settings.json @@ -0,0 +1,7 @@ +{ + "template": {}, + "version": 1, + "_meta": { + "description": "default settings for common Security Onion Detections indices" + } +} \ No newline at end of file From 881d6b313e394a56fc7e64c3de3302a6a1719b3e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 31 Jan 2024 17:04:11 -0500 Subject: [PATCH 021/140] Update VERSION - kilo --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9cf89c6c7..7f2e97617 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.50 +2.4.0-kilo From 49b5788ac14d4f79f6cac6b300e5f5cc6357172f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 1 Feb 2024 07:21:49 -0500 Subject: [PATCH 022/140] add bindings --- salt/soc/enabled.sls | 1 + salt/strelka/backend/enabled.sls | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 2661587f4..0eae0e80e 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -22,6 +22,7 @@ so-soc: - sobridge: - ipv4_address: {{ DOCKER.containers['so-soc'].ip }} - binds: + - /nsm/rules:/nsm/rules:rw #Need to tighten this up? - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /nsm/soc/uploads:/nsm/soc/uploads:rw - /opt/so/log/soc/:/opt/sensoroni/logs/:rw diff --git a/salt/strelka/backend/enabled.sls b/salt/strelka/backend/enabled.sls index fc56f4197..9ebb1a148 100644 --- a/salt/strelka/backend/enabled.sls +++ b/salt/strelka/backend/enabled.sls @@ -42,8 +42,8 @@ strelka_backend: {% endfor %} {% endif %} - restart_policy: on-failure - - watch: - - file: strelkarules + #- watch: + # - file: strelkarules delete_so-strelka-backend_so-status.disabled: file.uncomment: From fe196b56619ee240a81508a111e2e13427e3f382 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 1 Feb 2024 12:22:50 -0500 Subject: [PATCH 023/140] Add SOC Config for Detections --- salt/soc/defaults.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c1b9470c8..fdbdfd6b2 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1002,6 +1002,11 @@ soc: jobDir: jobs kratos: hostUrl: + elastalertengine: + communityRulesImportFrequencySeconds: 180 + elastAlertRulesFolder: /opt/so/rules/elastalert + rulesFingerprintFile: /opt/so/conf/soc/sigma.fingerprint + sigmaRulePackages: all elastic: hostUrl: remoteHostUrls: [] @@ -1043,6 +1048,15 @@ soc: - rbac/custom_roles userFiles: - rbac/users_roles + strelkaengine: + compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py + reposFolder: /nsm/rules/strelka/repos + rulesRepos: + - https://github.com/Security-Onion-Solutions/securityonion-yara + yaraRulesFolder: /opt/so/conf/strelka/rules + suricataengine: + communityRulesFile: /nsm/rules/suricata/emerging-all.rules + rulesFingerprintFile: /opt/so/conf/soc/emerging-all.fingerprint client: enableReverseLookup: false docsUrl: /docs/ From 8f81c9eb68b4c64dace40e065d91cf75b10019f0 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 2 Feb 2024 11:49:58 -0700 Subject: [PATCH 024/140] Updating config for Detection(s) --- salt/soc/defaults.yaml | 52 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index fdbdfd6b2..7f6686431 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -64,7 +64,7 @@ soc: icon: fa-external-link-alt target: _blank links: - - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' + - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' eventFields: default: - soc_timestamp @@ -1756,3 +1756,53 @@ soc: - amber+strict - red customEnabled: false + detections: + viewEnabled: true + createLink: /detection/create + eventFetchLimit: 500 + eventItemsPerPage: 50 + groupFetchLimit: 50 + mostRecentlyUsedLimit: 5 + safeStringMaxLength: 100 + queryBaseFilter: '_index:"*:so-detection" AND so_kind:detection' + eventFields: + default: + - so_detection.title + - so_detection.isEnabled + - so_detection.engine + - "@timestamp" + queries: + - name: "All Detections" + query: "_id:*" + - name: "Local Rules" + query: "so_detection.isCommunity:false" + - name: "Enabled" + query: "so_detection.isEnabled:true" + - name: "Disabled" + query: "so_detection.isEnabled:false" + - name: "Suricata" + query: "so_detection.engine:suricata" + - name: "ElastAlert" + query: "so_detection.engine:elastalert" + - name: "Strelka" + query: "so_detection.engine:strelka" + detection: + presets: + severity: + customEnabled: false + labels: + - unknown + - informational + - low + - medium + - high + - critical + engine: + customEnabled: false + labels: + - suricata + - elastalert + - strelka + severityTranslations: + minor: low + major: high From 378c99ae8844d78bf6691aa2179cc898e2de17d4 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 2 Feb 2024 18:25:54 -0500 Subject: [PATCH 025/140] Fix bindings --- salt/soc/defaults.yaml | 8 ++++---- salt/soc/enabled.sls | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7f6686431..6811529bf 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1004,8 +1004,8 @@ soc: hostUrl: elastalertengine: communityRulesImportFrequencySeconds: 180 - elastAlertRulesFolder: /opt/so/rules/elastalert - rulesFingerprintFile: /opt/so/conf/soc/sigma.fingerprint + elastAlertRulesFolder: /opt/sensoroni/elastalert + rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint sigmaRulePackages: all elastic: hostUrl: @@ -1053,10 +1053,10 @@ soc: reposFolder: /nsm/rules/strelka/repos rulesRepos: - https://github.com/Security-Onion-Solutions/securityonion-yara - yaraRulesFolder: /opt/so/conf/strelka/rules + yaraRulesFolder: /opt/sensoroni/yara suricataengine: communityRulesFile: /nsm/rules/suricata/emerging-all.rules - rulesFingerprintFile: /opt/so/conf/soc/emerging-all.fingerprint + rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint client: enableReverseLookup: false docsUrl: /docs/ diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 0eae0e80e..11f73e761 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -23,6 +23,9 @@ so-soc: - ipv4_address: {{ DOCKER.containers['so-soc'].ip }} - binds: - /nsm/rules:/nsm/rules:rw #Need to tighten this up? + - /opt/so/rules/yara:/opt/sensoroni/yara:rw + - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw + - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /nsm/soc/uploads:/nsm/soc/uploads:rw - /opt/so/log/soc/:/opt/sensoroni/logs/:rw From b7b501d289a749c2d8fd0afe246cba9a2cec551b Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 7 Feb 2024 15:02:52 -0500 Subject: [PATCH 026/140] Add Sigma pipelines --- salt/soc/final_sigma_pipeline.yaml | 7 +++++++ salt/soc/so_sigma_pipeline.yaml | 18 ++++++++++++++++++ salt/soc/soc_soc.yaml | 13 +++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 salt/soc/final_sigma_pipeline.yaml create mode 100644 salt/soc/so_sigma_pipeline.yaml diff --git a/salt/soc/final_sigma_pipeline.yaml b/salt/soc/final_sigma_pipeline.yaml new file mode 100644 index 000000000..656bfbb3e --- /dev/null +++ b/salt/soc/final_sigma_pipeline.yaml @@ -0,0 +1,7 @@ +name: Security Onion - Final Pipeline +priority: 95 +transformations: + - id: override_field_name_mapping + type: field_name_mapping + mapping: + FieldNameToOverride: NewFieldName diff --git a/salt/soc/so_sigma_pipeline.yaml b/salt/soc/so_sigma_pipeline.yaml new file mode 100644 index 000000000..a1c4d6d62 --- /dev/null +++ b/salt/soc/so_sigma_pipeline.yaml @@ -0,0 +1,18 @@ +name: Security Onion Baseline Pipeline +priority: 90 +transformations: + - id: baseline_field_name_mapping + type: field_name_mapping + mapping: + cs-method: http.method + c-uri: http.uri + c-useragent: http.useragent + cs-version: http.version + uid: user.uid + sid: rule.uuid + answer: answers + query: dns.query.name + src_ip: destination.ip.keyword + src_port: source.port + dst_ip: destination.ip.keyword + dst_port: destination.port \ No newline at end of file diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 0dd39620b..fe672fe3e 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -32,6 +32,14 @@ soc: global: True advanced: True helpLink: soc-customization.html + final_sigma_pipeline__yaml: + title: Final Sigma Pipeline + description: Final Processing Pipeline for Sigma Rules + syntax: yaml + file: True + global: True + advanced: True + helpLink: soc-customization.html config: licenseKey: title: License Key @@ -62,6 +70,11 @@ soc: global: True advanced: True modules: + elastalertengine: + sigmaRulePackages: + description: 'One of the following: core | core+ | core++ | all' + global: True + advanced: False elastic: index: description: Comma-separated list of indices or index patterns (wildcard "*" supported) that SOC will search for records. From 7e3187c0b8f4bba0c992dbf7719a661bc9924735 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 7 Feb 2024 15:35:31 -0500 Subject: [PATCH 027/140] Fixup sigma pipelines --- salt/soc/config.sls | 16 ++++++++++++++++ salt/soc/enabled.sls | 2 ++ .../{ => files/soc}/final_sigma_pipeline.yaml | 0 salt/soc/{ => files/soc}/so_sigma_pipeline.yaml | 0 salt/soc/soc_soc.yaml | 2 +- 5 files changed, 19 insertions(+), 1 deletion(-) rename salt/soc/{ => files/soc}/final_sigma_pipeline.yaml (100%) rename salt/soc/{ => files/soc}/so_sigma_pipeline.yaml (100%) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 95135566b..549bf94cf 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -57,6 +57,22 @@ socmotd: - mode: 600 - template: jinja +socsigmafinalpipeline: + file.managed: + - name: /opt/so/conf/soc/sigma_final_pipeline.yaml + - source: salt://soc/files/soc/sigma_final_pipeline.yaml + - user: 939 + - group: 939 + - mode: 600 + +socsigmasopipeline: + file.managed: + - name: /opt/so/conf/soc/sigma_so_pipeline.yaml + - source: salt://soc/files/soc/sigma_so_pipeline.yaml + - user: 939 + - group: 939 + - mode: 600 + socbanner: file.managed: - name: /opt/so/conf/soc/banner.md diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 11f73e761..535423179 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -32,6 +32,8 @@ so-soc: - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro + - /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro + - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:rw - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw diff --git a/salt/soc/final_sigma_pipeline.yaml b/salt/soc/files/soc/final_sigma_pipeline.yaml similarity index 100% rename from salt/soc/final_sigma_pipeline.yaml rename to salt/soc/files/soc/final_sigma_pipeline.yaml diff --git a/salt/soc/so_sigma_pipeline.yaml b/salt/soc/files/soc/so_sigma_pipeline.yaml similarity index 100% rename from salt/soc/so_sigma_pipeline.yaml rename to salt/soc/files/soc/so_sigma_pipeline.yaml diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index fe672fe3e..f413b5c73 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -32,7 +32,7 @@ soc: global: True advanced: True helpLink: soc-customization.html - final_sigma_pipeline__yaml: + sigma_final_pipeline__yaml: title: Final Sigma Pipeline description: Final Processing Pipeline for Sigma Rules syntax: yaml From 81a3e95914d602386a11e2f5289a98fe59f97ece Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 7 Feb 2024 16:42:16 -0500 Subject: [PATCH 028/140] Fixup sigma pipelines --- .../soc/{final_sigma_pipeline.yaml => sigma_final_pipeline.yaml} | 0 .../files/soc/{so_sigma_pipeline.yaml => sigma_so_pipeline.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename salt/soc/files/soc/{final_sigma_pipeline.yaml => sigma_final_pipeline.yaml} (100%) rename salt/soc/files/soc/{so_sigma_pipeline.yaml => sigma_so_pipeline.yaml} (100%) diff --git a/salt/soc/files/soc/final_sigma_pipeline.yaml b/salt/soc/files/soc/sigma_final_pipeline.yaml similarity index 100% rename from salt/soc/files/soc/final_sigma_pipeline.yaml rename to salt/soc/files/soc/sigma_final_pipeline.yaml diff --git a/salt/soc/files/soc/so_sigma_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml similarity index 100% rename from salt/soc/files/soc/so_sigma_pipeline.yaml rename to salt/soc/files/soc/sigma_so_pipeline.yaml From 29174566f3f7a8307a92eaac2073f108eddedc57 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 8 Feb 2024 09:44:56 -0700 Subject: [PATCH 029/140] WIP: Updated Detection Mappings, Changed Engine to Language Detection mappings updated to include the removal of Note and the addition of Tags, Ruleset, and Language. SOC defaults updated to use language based queries rather than engine and show the language column instead of the engine column in results. --- .../component/so/detection-mappings.json | 10 +++++++++- salt/soc/defaults.yaml | 18 +++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json index df53308f2..596236703 100644 --- a/salt/elasticsearch/templates/component/so/detection-mappings.json +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -47,13 +47,21 @@ "isCommunity": { "type": "boolean" }, - "note": { + "tags": { "type": "text" }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, "engine": { "ignore_above": 1024, "type": "keyword" }, + "language": { + "ignore_above": 1024, + "type": "keyword" + }, "overrides": { "properties": { "type": { diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6811529bf..cc6b417e6 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1769,7 +1769,7 @@ soc: default: - so_detection.title - so_detection.isEnabled - - so_detection.engine + - so_detection.language - "@timestamp" queries: - name: "All Detections" @@ -1781,11 +1781,11 @@ soc: - name: "Disabled" query: "so_detection.isEnabled:false" - name: "Suricata" - query: "so_detection.engine:suricata" - - name: "ElastAlert" - query: "so_detection.engine:elastalert" - - name: "Strelka" - query: "so_detection.engine:strelka" + query: "so_detection.language:suricata" + - name: "Sigma" + query: "so_detection.language:sigma" + - name: "Yara" + query: "so_detection.language:yara" detection: presets: severity: @@ -1797,12 +1797,12 @@ soc: - medium - high - critical - engine: + language: customEnabled: false labels: - suricata - - elastalert - - strelka + - sigma + - yara severityTranslations: minor: low major: high From 64f6d0fba93180f0627b56b5adeb822afa8c5193 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 9 Feb 2024 14:20:07 -0700 Subject: [PATCH 030/140] Updated Detection's ES Mappings Detection's now have a License field and the Comment model is defined now. --- .../component/so/detection-mappings.json | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json index 596236703..4efd2f73f 100644 --- a/salt/elasticsearch/templates/component/so/detection-mappings.json +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -62,6 +62,10 @@ "ignore_above": 1024, "type": "keyword" }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, "overrides": { "properties": { "type": { @@ -106,6 +110,24 @@ } } } + }, + "so_comment": { + "properties": { + "createTime": { + "type": "date" + }, + "detectionId": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "type": "text" + }, + "userId": { + "ignore_above": 1024, + "type": "keyword" + } + } } } } From 5a4e11b2f8f4182330f49ad9b290163e503216c6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 12 Feb 2024 16:09:47 -0500 Subject: [PATCH 031/140] Update soup Remove a function that isn't used any more --- salt/manager/tools/sbin/soup | 61 ------------------------------------ 1 file changed, 61 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 600cb5d4e..b572610ec 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -247,67 +247,6 @@ check_sudoers() { fi } -check_log_size_limit() { - local num_minion_pillars - num_minion_pillars=$(find /opt/so/saltstack/local/pillar/minions/ -type f | wc -l) - - if [[ $num_minion_pillars -gt 1 ]]; then - if find /opt/so/saltstack/local/pillar/minions/ -type f | grep -q "_heavynode"; then - lsl_msg='distributed' - fi - else - local minion_id - minion_id=$(lookup_salt_value "id" "" "grains" "" "local") - - local minion_arr - IFS='_' read -ra minion_arr <<< "$minion_id" - - local node_type="${minion_arr[0]}" - - local current_limit - # since it is possible for the salt-master service to be stopped when this is run, we need to check the pillar values locally - # we need to combine default local and default pillars before doing this so we can define --pillar-root in salt-call - local epoch_date=$(date +%s%N) - mkdir -vp /opt/so/saltstack/soup_tmp_${epoch_date}/ - cp -r /opt/so/saltstack/default/pillar/ /opt/so/saltstack/soup_tmp_${epoch_date}/ - # use \cp here to overwrite any pillar files from default with those in local for the tmp directory - \cp -r /opt/so/saltstack/local/pillar/ /opt/so/saltstack/soup_tmp_${epoch_date}/ - current_limit=$(salt-call pillar.get elasticsearch:log_size_limit --local --pillar-root=/opt/so/saltstack/soup_tmp_${epoch_date}/pillar --out=newline_values_only) - rm -rf /opt/so/saltstack/soup_tmp_${epoch_date}/ - - local percent - case $node_type in - 'standalone' | 'eval') - percent=50 - ;; - *) - percent=80 - ;; - esac - - local disk_dir="/" - if [ -d /nsm ]; then - disk_dir="/nsm" - fi - - local disk_size_1k - disk_size_1k=$(df $disk_dir | grep -v "^Filesystem" | awk '{print $2}') - - local ratio="1048576" - - local disk_size_gb - disk_size_gb=$( echo "$disk_size_1k" "$ratio" | awk '{print($1/$2)}' ) - - local new_limit - new_limit=$( echo "$disk_size_gb" "$percent" | awk '{printf("%.0f", $1 * ($2/100))}') - - if [[ $current_limit != "$new_limit" ]]; then - lsl_msg='single-node' - lsl_details=( "$current_limit" "$new_limit" "$minion_id" ) - fi - fi -} - check_os_updates() { # Check to see if there are OS updates echo "Checking for OS updates." From 510226944077b5ac460869d2eced99c0438ba169 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 12 Feb 2024 16:44:54 -0500 Subject: [PATCH 032/140] Update defaults --- salt/idstools/soc_idstools.yaml | 2 +- salt/soc/defaults.yaml | 2 +- salt/soc/soc_soc.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml index 634f68803..7cf11dba4 100644 --- a/salt/idstools/soc_idstools.yaml +++ b/salt/idstools/soc_idstools.yaml @@ -8,7 +8,7 @@ idstools: global: True helpLink: rules.html ruleset: - description: Defines the ruleset you want to run. Options are ETOPEN or ETPRO. + description: Defines the ruleset you want to run. Options are ETOPEN or ETPRO. -- WARNING -- Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides.' global: True regex: ETPRO\b|ETOPEN\b helpLink: rules.html diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index cc6b417e6..fd2eaf8c0 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1050,7 +1050,7 @@ soc: - rbac/users_roles strelkaengine: compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py - reposFolder: /nsm/rules/strelka/repos + reposFolder: /nsm/rules/yara/repos rulesRepos: - https://github.com/Security-Onion-Solutions/securityonion-yara yaraRulesFolder: /opt/sensoroni/yara diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index f413b5c73..fe0458820 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -72,7 +72,7 @@ soc: modules: elastalertengine: sigmaRulePackages: - description: 'One of the following: core | core+ | core++ | all' + description: 'Defines the Sigma Community Ruleset you want to run: core | core+ | core++ | all. -- WARNING -- Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides.' global: True advanced: False elastic: From ea80469c2db1bc690e26e4a7e5cf5c1afd44bc3d Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 12 Feb 2024 19:39:55 -0500 Subject: [PATCH 033/140] Detection Default queries --- salt/soc/defaults.yaml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index fd2eaf8c0..29cd7e1ac 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1770,21 +1770,23 @@ soc: - so_detection.title - so_detection.isEnabled - so_detection.language - - "@timestamp" + - so_detection.severity queries: - name: "All Detections" query: "_id:*" - - name: "Local Rules" + - name: "Custom Detections" query: "so_detection.isCommunity:false" - - name: "Enabled" + - name: "All Detections - Enabled" query: "so_detection.isEnabled:true" - - name: "Disabled" + - name: "All Detections - Disabled" query: "so_detection.isEnabled:false" - - name: "Suricata" + - name: "Detection Type - Suricata (NIDS)" query: "so_detection.language:suricata" - - name: "Sigma" + - name: "Detection Type - Sigma - All" query: "so_detection.language:sigma" - - name: "Yara" + - name: "Detection Type - Sigma - Windows" + query: 'so_detection.language:sigma AND so_detection.content: "*product: windows*"' + - name: "Detection Type - Yara (Strelka)" query: "so_detection.language:yara" detection: presets: From 0c6c6ba2d5d2e529ab6239f53ea6bd38d8a13446 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 13 Feb 2024 13:38:43 -0500 Subject: [PATCH 034/140] Various UI tweaks --- salt/soc/config.sls | 9 ++++++++- salt/soc/defaults.yaml | 6 +++--- salt/soc/enabled.sls | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 549bf94cf..e4dad8df2 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -9,9 +9,16 @@ include: - manager.sync_es_users +socdirtest: + file.directory: + - name: /opt/so/rules/elastalert/rules + - user: 939 + - group: 939 + - makedirs: True + socdir: file.directory: - - name: /opt/so/conf/soc + - name: /opt/so/conf/soc/fingerprints - user: 939 - group: 939 - makedirs: True diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 29cd7e1ac..c060698b4 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1006,7 +1006,7 @@ soc: communityRulesImportFrequencySeconds: 180 elastAlertRulesFolder: /opt/sensoroni/elastalert rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint - sigmaRulePackages: all + sigmaRulePackages: core elastic: hostUrl: remoteHostUrls: [] @@ -1050,10 +1050,10 @@ soc: - rbac/users_roles strelkaengine: compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py - reposFolder: /nsm/rules/yara/repos + reposFolder: /opt/sensoroni/yara/repos rulesRepos: - https://github.com/Security-Onion-Solutions/securityonion-yara - yaraRulesFolder: /opt/sensoroni/yara + yaraRulesFolder: /opt/sensoroni/yara/rules suricataengine: communityRulesFile: /nsm/rules/suricata/emerging-all.rules rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 535423179..7c04da825 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -23,7 +23,7 @@ so-soc: - ipv4_address: {{ DOCKER.containers['so-soc'].ip }} - binds: - /nsm/rules:/nsm/rules:rw #Need to tighten this up? - - /opt/so/rules/yara:/opt/sensoroni/yara:rw + - /opt/so/conf/strelka:/opt/sensoroni/yara:rw - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw - /nsm/soc/jobs:/opt/sensoroni/jobs:rw From 0d297274c8fc4f278e7b9e8530bdd39856425fb9 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 13 Feb 2024 12:53:18 -0700 Subject: [PATCH 035/140] DetectionComment Mapping Defined --- .../templates/component/so/detection-mappings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json index 4efd2f73f..9b68421e7 100644 --- a/salt/elasticsearch/templates/component/so/detection-mappings.json +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -111,7 +111,7 @@ } } }, - "so_comment": { + "so_detectioncomment": { "properties": { "createTime": { "type": "date" @@ -120,7 +120,7 @@ "ignore_above": 1024, "type": "keyword" }, - "description": { + "value": { "type": "text" }, "userId": { From 031ee078c528b614d398c6420cc5ab1f09bd45ac Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 19 Oct 2023 15:49:56 -0600 Subject: [PATCH 036/140] socsigmarepo Need write permissions on the /opt/so/rules dir so I can clone the sigma repo there. --- salt/soc/config.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 902d82ec7..95135566b 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -114,6 +114,13 @@ socuploaddir: - group: 939 - makedirs: True +socsigmarepo: + file.directory: + - name: /opt/so/rules + - user: 939 + - group: 939 + - mode: 775 + {% else %} {{sls}}_state_not_allowed: From 8800b7e8789297d72637a8a5016f14a1579c8db9 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 30 Jan 2024 15:43:51 -0700 Subject: [PATCH 037/140] WIP: Detections Changes Removed some strelka/yara rules from salt. Removed yara scripts for downloading and updating rules. This will be managed by SOC. Added a new compile_yara.py script. Added the strelka repos folder. --- salt/manager/init.sls | 53 +------------------ .../manager/tools/sbin_jinja/so-yara-download | 51 ------------------ salt/manager/tools/sbin_jinja/so-yara-update | 41 -------------- salt/soc/files/bin/compile_yara.py | 14 +++++ salt/strelka/backend/config.sls | 10 ---- salt/strelka/config.sls | 9 +++- 6 files changed, 24 insertions(+), 154 deletions(-) delete mode 100644 salt/manager/tools/sbin_jinja/so-yara-download delete mode 100755 salt/manager/tools/sbin_jinja/so-yara-update create mode 100644 salt/soc/files/bin/compile_yara.py diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 23ef189b5..51590a6ec 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -1,5 +1,5 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -61,7 +61,7 @@ manager_sbin: - user: 939 - group: 939 - file_mode: 755 - - exclude_pat: + - exclude_pat: - "*_test.py" yara_update_scripts: @@ -103,55 +103,6 @@ rules_dir: - group: socore - makedirs: True -{% if STRELKAMERGED.rules.enabled %} - -strelkarepos: - file.managed: - - name: /opt/so/conf/strelka/repos.txt - - source: salt://strelka/rules/repos.txt.jinja - - template: jinja - - defaults: - STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} - - makedirs: True - -strelka-yara-update: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-update >> /opt/so/log/yarasync/yara-update.log 2>&1' - - identifier: strelka-yara-update - - hour: '7' - - minute: '1' - -strelka-yara-download: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-download >> /opt/so/log/yarasync/yara-download.log 2>&1' - - identifier: strelka-yara-download - - hour: '7' - - minute: '1' - -{% if not GLOBALS.airgap %} -update_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-update - - onchanges: - - file: yara_update_scripts - -download_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-download - - onchanges: - - file: yara_update_scripts -{% endif %} -{% endif %} {% else %} {{sls}}_state_not_allowed: diff --git a/salt/manager/tools/sbin_jinja/so-yara-download b/salt/manager/tools/sbin_jinja/so-yara-download deleted file mode 100644 index aa9576253..000000000 --- a/salt/manager/tools/sbin_jinja/so-yara-download +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -NOROOT=1 -. /usr/sbin/so-common - -{%- set proxy = salt['pillar.get']('manager:proxy') %} -{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} - -# Download the rules from the internet -{%- if proxy %} -export http_proxy={{ proxy }} -export https_proxy={{ proxy }} -export no_proxy="{{ noproxy }}" -{%- endif %} - -repos="/opt/so/conf/strelka/repos.txt" -output_dir=/nsm/rules/yara -gh_status=$(curl -s -o /dev/null -w "%{http_code}" https://github.com) -clone_dir="/tmp" -if [ "$gh_status" == "200" ] || [ "$gh_status" == "301" ]; then - - while IFS= read -r repo; do - if ! $(echo "$repo" | grep -qE '^#'); then - # Remove old repo if existing bc of previous error condition or unexpected disruption - repo_name=`echo $repo | awk -F '/' '{print $NF}'` - [ -d $output_dir/$repo_name ] && rm -rf $output_dir/$repo_name - - # Clone repo and make appropriate directories for rules - git clone $repo $clone_dir/$repo_name - echo "Analyzing rules from $clone_dir/$repo_name..." - mkdir -p $output_dir/$repo_name - # Ensure a copy of the license is available for the rules - [ -f $clone_dir/$repo_name/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name - - # Copy over rules - for i in $(find $clone_dir/$repo_name -name "*.yar*"); do - rule_name=$(echo $i | awk -F '/' '{print $NF}') - cp $i $output_dir/$repo_name - done - rm -rf $clone_dir/$repo_name - fi - done < $repos - - echo "Done!" - -/usr/sbin/so-yara-update - -else - echo "Server returned $gh_status status code." - echo "No connectivity to Github...exiting..." - exit 1 -fi diff --git a/salt/manager/tools/sbin_jinja/so-yara-update b/salt/manager/tools/sbin_jinja/so-yara-update deleted file mode 100755 index 07c940f47..000000000 --- a/salt/manager/tools/sbin_jinja/so-yara-update +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -NOROOT=1 -. /usr/sbin/so-common - -echo "Starting to check for yara rule updates at $(date)..." - -newcounter=0 -excludedcounter=0 -excluded_rules=({{ EXCLUDEDRULES | join(' ') }}) - -# Pull down the SO Rules -SORULEDIR=/nsm/rules/yara -OUTPUTDIR=/opt/so/saltstack/local/salt/strelka/rules - -mkdir -p $OUTPUTDIR -# remove all rules prior to copy so we can clear out old rules -rm -f $OUTPUTDIR/* - -for i in $(find $SORULEDIR -name "*.yar" -o -name "*.yara"); do - rule_name=$(echo $i | awk -F '/' '{print $NF}') - if [[ ! "${excluded_rules[*]}" =~ ${rule_name} ]]; then - echo "Adding rule: $rule_name..." - cp $i $OUTPUTDIR/$rule_name - ((newcounter++)) - else - echo "Excluding rule: $rule_name..." - ((excludedcounter++)) - fi -done - -if [ "$newcounter" -gt 0 ] || [ "$excludedcounter" -gt 0 ];then - echo "$newcounter rules added." - echo "$excludedcounter rule(s) excluded." -fi - -echo "Finished rule updates at $(date)..." diff --git a/salt/soc/files/bin/compile_yara.py b/salt/soc/files/bin/compile_yara.py new file mode 100644 index 000000000..43c8b1a09 --- /dev/null +++ b/salt/soc/files/bin/compile_yara.py @@ -0,0 +1,14 @@ +import os +import yara +import glob +import sys + +def compile_yara_rules(rules_dir: str) -> None: + compiled_rules_path: str = os.path.join(rules_dir, "rules.yar.compiled") + rule_files: list[str] = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True) + + if rule_files: + rules: yara.Rules = yara.compile(filepaths={os.path.basename(f): f for f in rule_files}) + rules.save(compiled_rules_path) + +compile_yara_rules(sys.argv[1]) diff --git a/salt/strelka/backend/config.sls b/salt/strelka/backend/config.sls index d51debb1b..b39e06ac8 100644 --- a/salt/strelka/backend/config.sls +++ b/salt/strelka/backend/config.sls @@ -50,16 +50,6 @@ backend_taste: - user: 939 - group: 939 -{% if STRELKAMERGED.rules.enabled %} -strelkarules: - file.recurse: - - name: /opt/so/conf/strelka/rules - - source: salt://strelka/rules - - user: 939 - - group: 939 - - clean: True -{% endif %} - {% else %} {{sls}}_state_not_allowed: diff --git a/salt/strelka/config.sls b/salt/strelka/config.sls index 1d0f75adf..929bef113 100644 --- a/salt/strelka/config.sls +++ b/salt/strelka/config.sls @@ -1,5 +1,5 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -21,6 +21,13 @@ strelkarulesdir: - group: 939 - makedirs: True +strelkareposdir: + file.directory: + - name: /opt/so/conf/strelka/repos + - user: 939 + - group: 939 + - makedirs: True + strelkadatadir: file.directory: - name: /nsm/strelka From f321e734ebabe5043a01ab79d320abc485d3ef48 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 31 Jan 2024 10:39:47 -0700 Subject: [PATCH 038/140] Added so-detection mapping in elasticsearch --- salt/elasticsearch/defaults.yaml | 31 ++++- .../component/so/detection-mappings.json | 108 ++++++++++++++++++ .../component/so/detection-settings.json | 7 ++ 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 salt/elasticsearch/templates/component/so/detection-mappings.json create mode 100644 salt/elasticsearch/templates/component/so/detection-settings.json diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 03cd6d519..f4d8c8a95 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -198,6 +198,35 @@ elasticsearch: sort: field: '@timestamp' order: desc + so-detection: + index_sorting: false + index_template: + composed_of: + - detection-mappings + - detection-settings + index_patterns: + - so-detection* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + mapping: + total_fields: + limit: 1500 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc so-common: close: 30 delete: 365 @@ -8990,7 +9019,7 @@ elasticsearch: actions: set_priority: priority: 50 - min_age: 30d + min_age: 30d so-logs-ti_otx_x_threat: index_sorting: false index_template: diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json new file mode 100644 index 000000000..df53308f2 --- /dev/null +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -0,0 +1,108 @@ +{ + "template": { + "mappings": { + "properties": { + "so_audit_doc_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "@timestamp": { + "type": "date" + }, + "so_kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_operation": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_detection": { + "properties": { + "publicId": { + "type": "text" + }, + "title": { + "type": "text" + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + }, + "author": { + "type": "text" + }, + "description": { + "type": "text" + }, + "content": { + "type": "text" + }, + "isEnabled": { + "type": "boolean" + }, + "isReporting": { + "type": "boolean" + }, + "isCommunity": { + "type": "boolean" + }, + "note": { + "type": "text" + }, + "engine": { + "ignore_above": 1024, + "type": "keyword" + }, + "overrides": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "isEnabled": { + "type": "boolean" + }, + "createdAt": { + "type": "date" + }, + "updatedAt": { + "type": "date" + }, + "regex": { + "type": "text" + }, + "value": { + "type": "text" + }, + "thresholdType": { + "ignore_above": 1024, + "type": "keyword" + }, + "track": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "text" + }, + "count": { + "type": "long" + }, + "seconds": { + "type": "long" + }, + "customFilter": { + "type": "text" + } + } + } + } + } + } + } + }, + "_meta": { + "ecs_version": "1.12.2" + } +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/so/detection-settings.json b/salt/elasticsearch/templates/component/so/detection-settings.json new file mode 100644 index 000000000..7b0947a4c --- /dev/null +++ b/salt/elasticsearch/templates/component/so/detection-settings.json @@ -0,0 +1,7 @@ +{ + "template": {}, + "version": 1, + "_meta": { + "description": "default settings for common Security Onion Detections indices" + } +} \ No newline at end of file From c64f37ab671688e46d985806f7267f6c87ce48b0 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 15 Feb 2024 10:34:07 -0700 Subject: [PATCH 039/140] sigmaRulePackages is now a string array --- salt/soc/defaults.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index a19fb45f0..546114b9f 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -64,7 +64,7 @@ soc: icon: fa-external-link-alt target: _blank links: - - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' + - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' - name: actionProcessAncestors description: actionProcessAncestorsHelp icon: fa-people-roof @@ -1012,7 +1012,8 @@ soc: communityRulesImportFrequencySeconds: 180 elastAlertRulesFolder: /opt/sensoroni/elastalert rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint - sigmaRulePackages: core + sigmaRulePackages: + - core elastic: hostUrl: remoteHostUrls: [] From ffb3cc87b7adf47452ced95ae0d8aff370a1ef12 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 16 Feb 2024 11:55:10 -0500 Subject: [PATCH 040/140] Default ruleset; Descriptions --- salt/idstools/soc_idstools.yaml | 2 +- salt/soc/defaults.yaml | 1 + salt/soc/soc_soc.yaml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml index 7cf11dba4..f8ec3b8b6 100644 --- a/salt/idstools/soc_idstools.yaml +++ b/salt/idstools/soc_idstools.yaml @@ -8,7 +8,7 @@ idstools: global: True helpLink: rules.html ruleset: - description: Defines the ruleset you want to run. Options are ETOPEN or ETPRO. -- WARNING -- Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides.' + description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. WARNING! Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.' global: True regex: ETPRO\b|ETOPEN\b helpLink: rules.html diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 546114b9f..91d47cf91 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1014,6 +1014,7 @@ soc: rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint sigmaRulePackages: - core + - emerging_threats_addon elastic: hostUrl: remoteHostUrls: [] diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index fe0458820..fdfb09733 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -72,7 +72,7 @@ soc: modules: elastalertengine: sigmaRulePackages: - description: 'Defines the Sigma Community Ruleset you want to run: core | core+ | core++ | all. -- WARNING -- Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides.' + description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). WARNING! Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.' global: True advanced: False elastic: From 07fcfab7ec2392db3ecdead036d9deafb9edb6eb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 20 Feb 2024 10:14:11 -0500 Subject: [PATCH 041/140] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9cf89c6c7..5a99ed019 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.50 +2.4.60 From 4b314c871573144b89bf47fb45eebfc10bb3ba56 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 20 Feb 2024 10:30:09 -0500 Subject: [PATCH 042/140] replace correlate icon to avoid confusion with searcheng.in --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 244a021d3..dc836a9d7 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -20,7 +20,7 @@ soc: - dashboards - name: actionCorrelate description: actionCorrelateHelp - icon: fab fa-searchengin + icon: fa-magnifying-glass-arrow-right target: '' links: - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' From 6c6a362fcc14ae2c7df4c71ea4d2299072779d8a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 20 Feb 2024 19:14:18 -0500 Subject: [PATCH 043/140] add lock threads --- .github/workflows/lock-threads.yml | 42 ++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/lock-threads.yml diff --git a/.github/workflows/lock-threads.yml b/.github/workflows/lock-threads.yml new file mode 100644 index 000000000..25e5d8c17 --- /dev/null +++ b/.github/workflows/lock-threads.yml @@ -0,0 +1,42 @@ +name: 'Lock Threads' + +on: + schedule: + - cron: '50 1 * * *' + workflow_dispatch: + +permissions: + issues: write + pull-requests: write + discussions: write + +concurrency: + group: lock-threads + +jobs: + close-threads: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v5 + with: + days-before-issue-stale: -1 + days-before-issue-close: 60 + stale-issue-message: "This issue is stale because it has been inactive for an extended period. Stale issues convey that the issue, while important to someone, is not critical enough for the author, or other community members to work on, sponsor, or otherwise shepherd the issue through to a resolution." + close-issue-message: "This issue was closed because it has been stale for an extended period. It will be automatically locked in 30 days, after which no further commenting will be available." + days-before-pr-stale: 45 + days-before-pr-close: 60 + stale-pr-message: "This PR is stale because it has been inactive for an extended period. The longer a PR remains stale the more out of date with the main branch it becomes." + close-pr-message: "This PR was closed because it has been stale for an extended period. It will be automatically locked in 30 days. If there is still a commitment to finishing this PR re-open it before it is locked." + + lock-threads: + runs-on: ubuntu-latest + steps: + - uses: jertel/lock-threads@main + with: + include-discussion-currently-open: true + discussion-inactive-days: 90 + issue-inactive-days: 30 + pr-inactive-days: 30 From 9ca0f586ae63a90fef9c78b8b1fc24b020545ae3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 21 Feb 2024 11:45:02 -0500 Subject: [PATCH 044/140] Manage the repos --- salt/manager/files/mirror.txt | 0 salt/manager/files/repodownload.conf | 13 +++++++++++++ salt/manager/init.sls | 14 ++++++++++++++ setup/so-functions | 4 ++-- 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 salt/manager/files/mirror.txt create mode 100644 salt/manager/files/repodownload.conf diff --git a/salt/manager/files/mirror.txt b/salt/manager/files/mirror.txt new file mode 100644 index 000000000..e69de29bb diff --git a/salt/manager/files/repodownload.conf b/salt/manager/files/repodownload.conf new file mode 100644 index 000000000..3c156a9db --- /dev/null +++ b/salt/manager/files/repodownload.conf @@ -0,0 +1,13 @@ +[main] +gpgcheck=1 +installonly_limit=3 +clean_requirements_on_remove=True +best=True +skip_if_unavailable=False +cachedir=/opt/so/conf/reposync/cache +keepcache=0 +[securityonionsync] +name=Security Onion Repo repo +mirrorlist=file:///opt/so/conf/reposync/mirror.txt +enabled=1 +gpgcheck=1 \ No newline at end of file diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 23ef189b5..e51a448d5 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -75,6 +75,20 @@ yara_update_scripts: - defaults: EXCLUDEDRULES: {{ STRELKAMERGED.rules.excluded }} +so-repo-file: + file.managed: + - name: /opt/so/conf/reposync/repodownload.conf + - source: salt://manager/files/repodownload.conf + - user: socore + - group: socore + +so-repo-mirrorlist: + file.managed: + - name: /opt/so/conf/reposync/mirror.txt + - source: salt://manager/files/mirror.txt + - user: socore + - group: socore + so-repo-sync: {% if MANAGERMERGED.reposync.enabled %} cron.present: diff --git a/setup/so-functions b/setup/so-functions index f0462e4d6..ef1df4a71 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1811,8 +1811,8 @@ repo_sync_local() { info "Adding Repo Download Configuration" mkdir -p /nsm/repo mkdir -p /opt/so/conf/reposync/cache - echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9" > /opt/so/conf/reposync/mirror.txt - echo "https://so-repo-east.s3.us-east-005.backblazeb2.com/prod/2.4/oracle/9" >> /opt/so/conf/reposync/mirror.txt + echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9.3" > /opt/so/conf/reposync/mirror.txt + echo "https://repo-alt.securityonion.net/prod/2.4/oracle/9.3" >> /opt/so/conf/reposync/mirror.txt echo "[main]" > /opt/so/conf/reposync/repodownload.conf echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf echo "installonly_limit=3" >> /opt/so/conf/reposync/repodownload.conf From 25570e6ec2e9935c4a4481fd5f0ad32b9b7068de Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 21 Feb 2024 13:18:39 -0500 Subject: [PATCH 045/140] add missing template --- .github/DISCUSSION_TEMPLATE/2-4.yml | 190 ++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 .github/DISCUSSION_TEMPLATE/2-4.yml diff --git a/.github/DISCUSSION_TEMPLATE/2-4.yml b/.github/DISCUSSION_TEMPLATE/2-4.yml new file mode 100644 index 000000000..8e2592071 --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/2-4.yml @@ -0,0 +1,190 @@ +body: + - type: markdown + attributes: + value: | + ⚠️ This category is solely for conversations related to Security Onion 2.4 ⚠️ + + If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support + - type: dropdown + attributes: + label: Version + description: Which version of Security Onion 2.4.x are you asking about? + options: + - + - 2.4 Pre-release (Beta, Release Candidate) + - 2.4.10 + - 2.4.20 + - 2.4.30 + - 2.4.40 + - 2.4.50 + - 2.4.60 + - 2.4.70 + - 2.4.80 + - 2.4.90 + - 2.4.100 + - Other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Installation Method + description: How did you install Security Onion? + options: + - + - Security Onion ISO image + - Network installation on Red Hat derivative like Oracle, Rocky, Alma, etc. + - Network installation on Ubuntu + - Network installation on Debian + - Other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Description + description: > + Is this discussion about installation, configuration, upgrading, or other? + options: + - + - installation + - configuration + - upgrading + - other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Installation Type + description: > + When you installed, did you choose Import, Eval, Standalone, Distributed, or something else? + options: + - + - Import + - Eval + - Standalone + - Distributed + - other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Location + description: > + Is this deployment in the cloud, on-prem with Internet access, or airgap? + options: + - + - cloud + - on-prem with Internet access + - airgap + - other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Hardware Specs + description: > + Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://docs.securityonion.net/en/2.4/hardware.html? + options: + - + - Meets minimum requirements + - Exceeds minimum requirements + - Does not meet minimum requirements + - other (please provide detail below) + validations: + required: true + - type: input + attributes: + label: CPU + description: How many CPU cores do you have? + validations: + required: true + - type: input + attributes: + label: RAM + description: How much RAM do you have? + validations: + required: true + - type: input + attributes: + label: Storage for / + description: How much storage do you have for the / partition? + validations: + required: true + - type: input + attributes: + label: Storage for /nsm + description: How much storage do you have for the /nsm partition? + validations: + required: true + - type: dropdown + attributes: + label: Network Traffic Collection + description: > + Are you collecting network traffic from a tap or span port? + options: + - + - tap + - span port + - other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Network Traffic Speeds + description: > + How much network traffic are you monitoring? + options: + - + - Less than 1Gbps + - 1Gbps to 10Gbps + - more than 10Gbps + validations: + required: true + - type: dropdown + attributes: + label: Status + description: > + Does SOC Grid show all services on all nodes as running OK? + options: + - + - Yes, all services on all nodes are running OK + - No, one or more services are failed (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Salt Status + description: > + Do you get any failures when you run "sudo salt-call state.highstate"? + options: + - + - Yes, there are salt failures (please provide detail below) + - No, there are no failures + validations: + required: true + - type: dropdown + attributes: + label: Logs + description: > + Are there any additional clues in /opt/so/log/? + options: + - + - Yes, there are additional clues in /opt/so/log/ (please provide detail below) + - No, there are no additional clues + validations: + required: true + - type: textarea + attributes: + label: Detail + description: Please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and then provide detailed information to help us help you. + placeholder: |- + STOP! Before typing, please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 in their entirety! + + If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support + validations: + required: true + - type: checkboxes + attributes: + label: Guidelines + options: + - label: I have read the discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and assert that I have followed the guidelines. + required: true From 162785575cbae9d9454a1d58830ab740129f1c57 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Feb 2024 15:28:24 -0500 Subject: [PATCH 046/140] nest under policy --- salt/elasticsearch/soc_elasticsearch.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 9a64190b3..e68d0441b 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -95,6 +95,7 @@ elasticsearch: description: The order to sort by. Must set index_sorting to True. global: True helpLink: elasticsearch.html + policy: phases: hot: max_age: From 927ea0c9ecf01e4f8b08b6e2688abbefac8cf1d2 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 21 Feb 2024 15:56:12 -0500 Subject: [PATCH 047/140] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7f2e97617..5a99ed019 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-kilo +2.4.60 From 0a9022ba6a1f6ce82efa28d632b2f4f4bb1cd7b6 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 21 Feb 2024 17:07:08 -0500 Subject: [PATCH 048/140] Add hash mappings --- salt/soc/files/soc/sigma_so_pipeline.yaml | 32 ++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index a1c4d6d62..54ce83eff 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -15,4 +15,34 @@ transformations: src_ip: destination.ip.keyword src_port: source.port dst_ip: destination.ip.keyword - dst_port: destination.port \ No newline at end of file + dst_port: destination.port + - id: hashes_process-creation + type: field_name_mapping + mapping: + winlog.event_data.sha256: process.hash.sha256 + winlog.event_data.sha1: process.hash.sha1 + winlog.event_data.md5: process.hash.md5 + rule_conditions: + - type: logsource + product: windows + category: process_creation + - id: hashes_image-load + type: field_name_mapping + mapping: + winlog.event_data.sha256: dll.hash.sha256 + winlog.event_data.sha1: dll.hash.sha1 + winlog.event_data.md5: dll.hash.md5 + rule_conditions: + - type: logsource + product: windows + category: image_load + - id: hashes_driver-load + type: field_name_mapping + mapping: + winlog.event_data.sha256: dll.hash.sha256 + winlog.event_data.sha1: dll.hash.sha1 + winlog.event_data.md5: dll.hash.md5 + rule_conditions: + - type: logsource + product: windows + category: driver_load \ No newline at end of file From c886e7279363a8f0c614dc8a753166f5d418d5dc Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 22 Feb 2024 08:59:33 -0500 Subject: [PATCH 049/140] Imphash mappings --- salt/soc/files/soc/sigma_so_pipeline.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 54ce83eff..8121a6f13 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -15,13 +15,15 @@ transformations: src_ip: destination.ip.keyword src_port: source.port dst_ip: destination.ip.keyword - dst_port: destination.port + dst_port: destination.port + winlog.event_data.User: user.name - id: hashes_process-creation type: field_name_mapping mapping: winlog.event_data.sha256: process.hash.sha256 winlog.event_data.sha1: process.hash.sha1 winlog.event_data.md5: process.hash.md5 + winlog.event_data.Imphash: process.pe.imphash rule_conditions: - type: logsource product: windows @@ -32,6 +34,7 @@ transformations: winlog.event_data.sha256: dll.hash.sha256 winlog.event_data.sha1: dll.hash.sha1 winlog.event_data.md5: dll.hash.md5 + winlog.event_data.Imphash: dll.pe.imphash rule_conditions: - type: logsource product: windows @@ -42,6 +45,7 @@ transformations: winlog.event_data.sha256: dll.hash.sha256 winlog.event_data.sha1: dll.hash.sha1 winlog.event_data.md5: dll.hash.md5 + winlog.event_data.Imphash: dll.pe.imphash rule_conditions: - type: logsource product: windows From 759b2ff59e5ffa201464a32a0ac776925a8044bb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 22 Feb 2024 10:03:51 -0500 Subject: [PATCH 050/140] Manage the repos --- salt/manager/files/mirror.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/files/mirror.txt b/salt/manager/files/mirror.txt index e69de29bb..732c116b4 100644 --- a/salt/manager/files/mirror.txt +++ b/salt/manager/files/mirror.txt @@ -0,0 +1,2 @@ +https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9 +https://repo-alt.securityonion.net/prod/2.4/oracle/9 \ No newline at end of file From e7914fc5a16bc47d243835244985088e717d9a80 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 22 Feb 2024 12:49:06 -0500 Subject: [PATCH 051/140] Update stenoloss.sh --- salt/telegraf/scripts/stenoloss.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/telegraf/scripts/stenoloss.sh b/salt/telegraf/scripts/stenoloss.sh index 5c27ee7a5..5219dcfd0 100644 --- a/salt/telegraf/scripts/stenoloss.sh +++ b/salt/telegraf/scripts/stenoloss.sh @@ -10,8 +10,8 @@ # if this script isn't already running if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then - CHECKIT=$(grep "Thread 0" /var/log/stenographer/stenographer.log |tac |head -2|wc -l) - STENOGREP=$(grep "Thread 0" /var/log/stenographer/stenographer.log |tac |head -2) + CHECKIT=$(grep "Thread 0 stats" /var/log/stenographer/stenographer.log |tac |head -2|wc -l) + STENOGREP=$(grep "Thread 0 stats" /var/log/stenographer/stenographer.log |tac |head -2) declare RESULT=($STENOGREP) From d04aa06455e7e96b5cbdac6c09953e01bd718c81 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 22 Feb 2024 14:01:02 -0500 Subject: [PATCH 052/140] Fix source.ip --- salt/soc/files/soc/sigma_so_pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 8121a6f13..533823e6f 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -12,7 +12,7 @@ transformations: sid: rule.uuid answer: answers query: dns.query.name - src_ip: destination.ip.keyword + src_ip: source.ip.keyword src_port: source.port dst_ip: destination.ip.keyword dst_port: destination.port From b8baca417bc6a50a5149b1f538dba6e9eb26ffb4 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 23 Feb 2024 14:03:04 -0500 Subject: [PATCH 053/140] add endpoint_x_events_x_process to defaults.yaml --- salt/soc/defaults.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 2c15fe996..d672d1dad 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -995,6 +995,14 @@ soc: - tds.header_type - log.id.uid - event.dataset + ':endpoint:endpoint_x_events_x_process': + - soc_timestamp + - event.dataset + - host.name + - user.name + - process.parent.name + - process.name + - process.working_directory server: bindAddress: 0.0.0.0:9822 baseUrl: / From 573d565976b267b304ce6c03b5ca8a9041b4ad70 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 23 Feb 2024 15:03:44 -0500 Subject: [PATCH 054/140] convert _x_ to . for soc ui to config --- salt/soc/merged.map.jinja | 2 ++ salt/soc/soc_soc.yaml | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index 33c0070ad..55d8d2600 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -66,6 +66,8 @@ {% do SOCMERGED.config.server.client.alerts.update({'actions': standard_actions}) %} {% do SOCMERGED.config.server.client.cases.update({'actions': standard_actions}) %} +{# replace the _x_ with . for soc ui to config conversion #} +{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.process': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_process') }) %} {% set standard_eventFields = SOCMERGED.config.pop('eventFields') %} {% do SOCMERGED.config.server.client.hunt.update({'eventFields': standard_eventFields}) %} {% do SOCMERGED.config.server.client.dashboards.update({'eventFields': standard_eventFields}) %} diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index fdfb09733..13e50a0cb 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -55,10 +55,11 @@ soc: global: True forcedType: "[]{}" eventFields: - default: - description: Event fields mappings are defined by the format ":event.module:event.dataset". For example, to customize which fields show for 'syslog' events originating from 'zeek', find the eventField item in the left panel that looks like ':zeek:syslog'. This 'default' entry is used for all events that do not match an existing mapping defined in the list to the left. + default: &eventFields + description: Event fields mappings are defined by the format ":event.module:event.dataset". For example, to customize which fields show for 'syslog' events originating from 'zeek', find the eventField item in the left panel that looks like ':zeek:syslog'. The 'default' entry is used for all events that do not match an existing mapping defined in the list to the left. global: True advanced: True + ':endpoint:endpoint_x_events_x_process': *eventFields server: srvKey: description: Unique key for protecting the integrity of user submitted data via the web browser. From 7da0ccf5a628d71e75eef7fda547004ca693fec4 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 23 Feb 2024 15:35:53 -0500 Subject: [PATCH 055/140] add more endpoint.events.x entries to merged.map.jinja --- salt/soc/merged.map.jinja | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index 55d8d2600..c0ea836e5 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -67,7 +67,13 @@ {% do SOCMERGED.config.server.client.cases.update({'actions': standard_actions}) %} {# replace the _x_ with . for soc ui to config conversion #} +{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.api': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_api') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.file': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_file') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.library': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_library') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.network': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_network') }) %} {% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.process': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_process') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.registry': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_registry') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.security': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_security') }) %} {% set standard_eventFields = SOCMERGED.config.pop('eventFields') %} {% do SOCMERGED.config.server.client.hunt.update({'eventFields': standard_eventFields}) %} {% do SOCMERGED.config.server.client.dashboards.update({'eventFields': standard_eventFields}) %} From b7ef1e8af121862174fd72b0a75125fbf1341ec1 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 23 Feb 2024 15:38:53 -0500 Subject: [PATCH 056/140] add more endpoint.events.x fields to soc_soc.yaml --- salt/soc/soc_soc.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 13e50a0cb..f59d6117b 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -59,7 +59,13 @@ soc: description: Event fields mappings are defined by the format ":event.module:event.dataset". For example, to customize which fields show for 'syslog' events originating from 'zeek', find the eventField item in the left panel that looks like ':zeek:syslog'. The 'default' entry is used for all events that do not match an existing mapping defined in the list to the left. global: True advanced: True + ':endpoint:endpoint_x_events_x_api': *eventFields + ':endpoint:endpoint_x_events_x_file': *eventFields + ':endpoint:endpoint_x_events_x_library': *eventFields + ':endpoint:endpoint_x_events_x_network': *eventFields ':endpoint:endpoint_x_events_x_process': *eventFields + ':endpoint:endpoint_x_events_x_registry': *eventFields + ':endpoint:endpoint_x_events_x_security': *eventFields server: srvKey: description: Unique key for protecting the integrity of user submitted data via the web browser. From 58f4fb87d08813ab41eea5b24e6b2a4a2326d5ff Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 23 Feb 2024 17:06:29 -0500 Subject: [PATCH 057/140] fix new eventFields in soc_soc.yaml --- salt/soc/soc_soc.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index f59d6117b..a9d36c70c 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -59,13 +59,13 @@ soc: description: Event fields mappings are defined by the format ":event.module:event.dataset". For example, to customize which fields show for 'syslog' events originating from 'zeek', find the eventField item in the left panel that looks like ':zeek:syslog'. The 'default' entry is used for all events that do not match an existing mapping defined in the list to the left. global: True advanced: True - ':endpoint:endpoint_x_events_x_api': *eventFields - ':endpoint:endpoint_x_events_x_file': *eventFields - ':endpoint:endpoint_x_events_x_library': *eventFields - ':endpoint:endpoint_x_events_x_network': *eventFields - ':endpoint:endpoint_x_events_x_process': *eventFields - ':endpoint:endpoint_x_events_x_registry': *eventFields - ':endpoint:endpoint_x_events_x_security': *eventFields + ':endpoint:events_x_api': *eventFields + ':endpoint:events_x_file': *eventFields + ':endpoint:events_x_library': *eventFields + ':endpoint:events_x_network': *eventFields + ':endpoint:events_x_process': *eventFields + ':endpoint:events_x_registry': *eventFields + ':endpoint:events_x_security': *eventFields server: srvKey: description: Unique key for protecting the integrity of user submitted data via the web browser. From daf96d79342959155d2fccb7313c8e5a9b393f60 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 23 Feb 2024 17:07:48 -0500 Subject: [PATCH 058/140] fix new eventFields in merged.map.jinja --- salt/soc/merged.map.jinja | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index c0ea836e5..65091158e 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -67,13 +67,13 @@ {% do SOCMERGED.config.server.client.cases.update({'actions': standard_actions}) %} {# replace the _x_ with . for soc ui to config conversion #} -{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.api': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_api') }) %} -{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.file': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_file') }) %} -{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.library': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_library') }) %} -{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.network': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_network') }) %} -{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.process': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_process') }) %} -{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.registry': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_registry') }) %} -{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.security': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_security') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:events.api': SOCMERGED.config.eventFields.pop(':endpoint:events_x_api') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:events.file': SOCMERGED.config.eventFields.pop(':endpoint:events_x_file') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:events.library': SOCMERGED.config.eventFields.pop(':endpoint:events_x_library') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:events.network': SOCMERGED.config.eventFields.pop(':endpoint:events_x_network') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:events.process': SOCMERGED.config.eventFields.pop(':endpoint:events_x_process') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:events.registry': SOCMERGED.config.eventFields.pop(':endpoint:events_x_registry') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:events.security': SOCMERGED.config.eventFields.pop(':endpoint:events_x_security') }) %} {% set standard_eventFields = SOCMERGED.config.pop('eventFields') %} {% do SOCMERGED.config.server.client.hunt.update({'eventFields': standard_eventFields}) %} {% do SOCMERGED.config.server.client.dashboards.update({'eventFields': standard_eventFields}) %} From d6cb8ab92823b95d0d35cee157549b52a91d2c97 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 23 Feb 2024 17:09:40 -0500 Subject: [PATCH 059/140] update events_x_process in defaults.yaml --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d672d1dad..7204027fc 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -995,7 +995,7 @@ soc: - tds.header_type - log.id.uid - event.dataset - ':endpoint:endpoint_x_events_x_process': + ':endpoint:events_x_process': - soc_timestamp - event.dataset - host.name From a6bb7216f9aaead1b51887cee531e8b74e6272c2 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 26 Feb 2024 08:18:42 -0500 Subject: [PATCH 060/140] Add Detection AutoUpdate config --- salt/soc/defaults.yaml | 2 ++ salt/soc/soc_soc.yaml | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 2c15fe996..5267955b9 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1009,6 +1009,7 @@ soc: kratos: hostUrl: elastalertengine: + autoUpdateEnabled: false communityRulesImportFrequencySeconds: 180 elastAlertRulesFolder: /opt/sensoroni/elastalert rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint @@ -1057,6 +1058,7 @@ soc: userFiles: - rbac/users_roles strelkaengine: + autoUpdateEnabled: false compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py reposFolder: /opt/sensoroni/yara/repos rulesRepos: diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index fdfb09733..74ae1051b 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -75,6 +75,10 @@ soc: description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). WARNING! Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.' global: True advanced: False + autoUpdateEnabled: + description: 'Set to true to enable automatic updates of the Sigma Community Ruleset.' + global: True + advanced: True elastic: index: description: Comma-separated list of indices or index patterns (wildcard "*" supported) that SOC will search for records. @@ -133,6 +137,11 @@ soc: description: Duration (in milliseconds) to wait for a response from the Salt API when executing common grid management tasks before giving up and showing an error on the SOC UI. global: True advanced: True + strelkaengine: + autoUpdateEnabled: + description: 'Set to true to enable automatic updates of the Yara ruleset.' + global: True + advanced: True client: enableReverseLookup: description: Set to true to enable reverse DNS lookups for IP addresses in the SOC UI. From ca249312baddc9c3d6b7321f104ccc58da447acb Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 26 Feb 2024 09:38:14 -0500 Subject: [PATCH 061/140] FEATURE: Add new SOC action for Process Info #12421 --- salt/soc/defaults.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7204027fc..20f9c284f 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -65,12 +65,18 @@ soc: target: _blank links: - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' + - name: actionProcessInfo + description: actionProcessInfoHelp + icon: fa-person-running + target: '' + links: + - '/#/hunt?q=(process.entity_id:"{:process.entity_id}") | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path' - name: actionProcessAncestors description: actionProcessAncestorsHelp icon: fa-people-roof target: '' links: - - '/#/hunt?q=(process.entity_id:"{:process.entity_id}" OR process.entity_id:"{:process.Ext.ancestry|processAncestors}") | groupby process.parent.name | groupby -sankey process.parent.name process.name | groupby process.name | groupby event.module event.dataset | table soc_timestamp event.dataset host.name user.name process.parent.name process.name process.working_directory' + - '/#/hunt?q=(process.entity_id:"{:process.entity_id}" OR process.entity_id:"{:process.Ext.ancestry|processAncestors}") | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.parent.name | groupby -sankey process.parent.name process.name | groupby process.name | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path' eventFields: default: - soc_timestamp From 4df21148fc4b7117b6a69b2914d961855d3f12e7 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 26 Feb 2024 09:40:51 -0500 Subject: [PATCH 062/140] FEATURE: Add default columns for endpoint.events datasets #12425 --- salt/soc/defaults.yaml | 57 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 20f9c284f..b5fe0e626 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1001,14 +1001,69 @@ soc: - tds.header_type - log.id.uid - event.dataset + ':endpoint:events_x_api': + - soc_timestamp + - host.name + - user.name + - process.name + - process.Ext.api.name + - process.thread.Ext.call_stack_final_user_module.path + - event.dataset + ':endpoint:events_x_file': + - soc_timestamp + - host.name + - user.name + - process.name + - event.action + - file.path + - event.dataset + ':endpoint:events_x_library': + - soc_timestamp + - host.name + - user.name + - process.name + - event.action + - dll.path + - dll.code_signature.status + - dll.code_signature.subject_name + - event.dataset + ':endpoint:events_x_network': + - soc_timestamp + - host.name + - user.name + - process.name + - event.action + - source.ip + - source.port + - destination.ip + - destination.port + - network.community_id + - event.dataset ':endpoint:events_x_process': - soc_timestamp - - event.dataset - host.name - user.name - process.parent.name - process.name + - event.action - process.working_directory + - event.dataset + ':endpoint:events_x_registry': + - soc_timestamp + - host.name + - user.name + - process.name + - event.action + - registry.path + - event.dataset + ':endpoint:events_x_security': + - soc_timestamp + - host.name + - user.name + - process.executable + - event.action + - event.outcome + - event.dataset server: bindAddress: 0.0.0.0:9822 baseUrl: / From c8a95a87069a996543672c50485280394e45226d Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 26 Feb 2024 09:59:07 -0500 Subject: [PATCH 063/140] FEATURE: Add new endpoint dashboards #12428 --- salt/soc/defaults.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index b5fe0e626..c1b3ebabb 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1502,13 +1502,22 @@ soc: query: 'event.category: network AND _exists_:process.executable AND (_exists_:dns.question.name OR _exists_:dns.answers.data) | groupby -sankey host.name dns.question.name | groupby event.dataset event.type | groupby host.name | groupby process.executable | groupby dns.question.name | groupby dns.answers.data' - name: Host Process Activity description: Process activity captured on an endpoint - query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable | table soc_timestamp event.dataset host.name user.name process.parent.name process.name process.working_directory' + query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable' - name: Host File Activity description: File activity captured on an endpoint query: 'event.category: file AND _exists_:process.executable | groupby -sankey host.name process.executable | groupby host.name | groupby event.dataset event.action event.type | groupby file.name | groupby process.executable' - name: Host Network & Process Mappings description: Network activity mapped to originating processes query: 'event.category: network AND _exists_:process.executable | groupby -sankey event.action host.name | groupby -sankey host.name user.name | groupby event.dataset* event.type* event.action* | groupby host.name | groupby user.name | groupby dns.question.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby process.name | groupby source.ip | groupby destination.ip | groupby destination.port' + - name: Host API Events + description: API (Application Programming Interface) events from endpoints + query: 'event.dataset:endpoint.events.api | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby process.name | groupby process.Ext.api.name' + - name: Host Library Events + description: Library events from endpoints + query: 'event.dataset:endpoint.events.library | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby process.name | groupby event.action | groupby dll.path | groupby dll.code_signature.status | groupby dll.code_signature.subject_name' + - name: Host Security Events + description: Security events from endpoints + query: 'event.dataset:endpoint.events.security | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby process.executable | groupby event.action | groupby event.outcome' - name: Strelka description: Strelka file analysis query: 'event.module:strelka | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby file.name' From 9a7e2153eedec1fbeb61df3db918ba5b7e7baa39 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 26 Feb 2024 11:01:53 -0500 Subject: [PATCH 064/140] add classification.config --- salt/suricata/classification/classification.config | 2 ++ salt/suricata/config.sls | 7 +++++++ salt/suricata/enabled.sls | 1 + salt/suricata/soc_suricata.yaml | 7 +++++++ 4 files changed, 17 insertions(+) create mode 100644 salt/suricata/classification/classification.config diff --git a/salt/suricata/classification/classification.config b/salt/suricata/classification/classification.config new file mode 100644 index 000000000..69918fed7 --- /dev/null +++ b/salt/suricata/classification/classification.config @@ -0,0 +1,2 @@ +# configuration classification: shortname,description,priority +# configuration classification: misc-activity,Misc activity,3 diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 3ec1324bf..00364f384 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -129,6 +129,13 @@ surithresholding: - group: 940 - template: jinja +suriclassifications: + file.managed: + - name: /opt/so/conf/suricata/classification.config + - source: salt://suricata/classification/classification.config + - user: 940 + - group: 940 + # BPF compilation and configuration {% if SURICATABPF %} {% set BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + SURICATABPF|join(" "),cwd='/root') %} diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index ce309e41a..f96472ae2 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -27,6 +27,7 @@ so-suricata: - binds: - /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro - /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro + - /opt/so/conf/suricata/classification.config:/etc/suricata/classification.config:ro - /opt/so/conf/suricata/rules:/etc/suricata/rules:ro - /opt/so/log/suricata/:/var/log/suricata/:rw - /nsm/suricata/:/nsm/:rw diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 30f277c0a..4fd720ef1 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -11,6 +11,13 @@ suricata: multiline: True title: SIDS helpLink: suricata.html + classification: + classification__config: + description: Classifications config file. + file: True + global: True + multiline: True + helpLink: suricata.html config: af-packet: interface: From f8424f3dad29bed27f0d02006cb3af14f5f76e39 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 26 Feb 2024 11:22:09 -0500 Subject: [PATCH 065/140] Update defaults.yaml --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c1b3ebabb..d86262fe7 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1502,7 +1502,7 @@ soc: query: 'event.category: network AND _exists_:process.executable AND (_exists_:dns.question.name OR _exists_:dns.answers.data) | groupby -sankey host.name dns.question.name | groupby event.dataset event.type | groupby host.name | groupby process.executable | groupby dns.question.name | groupby dns.answers.data' - name: Host Process Activity description: Process activity captured on an endpoint - query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable' + query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable | table soc_timestamp host.name user.name process.parent.name process.name event.action process.working_directory event.dataset' - name: Host File Activity description: File activity captured on an endpoint query: 'event.category: file AND _exists_:process.executable | groupby -sankey host.name process.executable | groupby host.name | groupby event.dataset event.action event.type | groupby file.name | groupby process.executable' From 1d099f97d2577077607152ae2579e974936db53a Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 26 Feb 2024 11:27:56 -0500 Subject: [PATCH 066/140] Update pattern for endpoint diagnostic template --- salt/elasticsearch/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index f4d8c8a95..8e28f2e41 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3926,7 +3926,7 @@ elasticsearch: allow_custom_routing: false hidden: false index_patterns: - - logs-endpoint.diagnostic.collection-* + - .logs-endpoint.diagnostic.collection-* priority: 501 template: settings: From 466dac30bbf0f7b6d3d2d065ea8eed3639541ac8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 26 Feb 2024 12:15:17 -0500 Subject: [PATCH 067/140] soup for classifications --- salt/manager/tools/sbin/soup | 15 +++++++++++++++ salt/suricata/soc_suricata.yaml | 1 + 2 files changed, 16 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 90ec636ef..752ae6e21 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -356,6 +356,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.20 ]] && up_to_2.4.30 [[ "$INSTALLEDVERSION" == 2.4.30 ]] && up_to_2.4.40 [[ "$INSTALLEDVERSION" == 2.4.40 ]] && up_to_2.4.50 + [[ "$INSTALLEDVERSION" == 2.4.50 ]] && up_to_2.4.60 true } @@ -371,6 +372,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.20 ]] && post_to_2.4.30 [[ "$POSTVERSION" == 2.4.30 ]] && post_to_2.4.40 [[ "$POSTVERSION" == 2.4.40 ]] && post_to_2.4.50 + [[ "$POSTVERSION" == 2.4.50 ]] && post_to_2.4.60 true } @@ -427,6 +429,11 @@ post_to_2.4.50() { POSTVERSION=2.4.50 } +post_to_2.4.60() { + echo "Nothing to apply" + POSTVERSION=2.4.60 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -556,6 +563,14 @@ up_to_2.4.50() { INSTALLEDVERSION=2.4.50 } +up_to_2.4.60() { + echo "Creating directory to store Suricata classification.config" + mkdir -vp /opt/so/saltstack/local/salt/suricata/classification + chown socore:socore /opt/so/saltstack/local/salt/suricata/classification + + INSTALLEDVERSION=2.4.60 +} + determine_elastic_agent_upgrade() { if [[ $is_airgap -eq 0 ]]; then update_elastic_agent_airgap diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 4fd720ef1..b54a44cbc 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -17,6 +17,7 @@ suricata: file: True global: True multiline: True + title: Classifications helpLink: suricata.html config: af-packet: From 8b7f7933bdfa4e67aa7e3a84dd1c3d03340935c7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 26 Feb 2024 15:29:13 -0500 Subject: [PATCH 068/140] suricata container watch classification.config --- salt/suricata/classification/classification.config | 4 ++-- salt/suricata/enabled.sls | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/suricata/classification/classification.config b/salt/suricata/classification/classification.config index 69918fed7..e597eb5a1 100644 --- a/salt/suricata/classification/classification.config +++ b/salt/suricata/classification/classification.config @@ -1,2 +1,2 @@ -# configuration classification: shortname,description,priority -# configuration classification: misc-activity,Misc activity,3 +# config classification: shortname,description,priority +# config classification: misc-activity,Misc activity,3 diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index f96472ae2..94b95ff5d 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -50,10 +50,12 @@ so-suricata: - file: surithresholding - file: /opt/so/conf/suricata/rules/ - file: /opt/so/conf/suricata/bpf + - file: suriclassifications - require: - file: suriconfig - file: surithresholding - file: suribpf + - file: suriclassifications delete_so-suricata_so-status.disabled: file.uncomment: From c6baa4be1baad88a7301e7c08d8c960d865afca0 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 26 Feb 2024 16:19:32 -0500 Subject: [PATCH 069/140] Airgap Support - Detections module --- salt/manager/tools/sbin/soup | 10 ++++++---- salt/soc/enabled.sls | 2 +- salt/soc/merged.map.jinja | 6 ++++++ setup/so-setup | 6 ++---- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 90ec636ef..655e99f6c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -603,6 +603,10 @@ update_airgap_rules() { if [ -d /nsm/repo/rules/sigma ]; then rsync -av $UPDATE_DIR/agrules/sigma/* /nsm/repo/rules/sigma/ fi + + # SOC Detections Airgap + rsync -av $UPDATE_DIR/agrules/detect-sigma/* /nsm/rules/detect-sigma/ + rsync -av $UPDATE_DIR/agrules/detect-yara/* /nsm/rules/detect-yara/ } update_airgap_repo() { @@ -931,10 +935,8 @@ main() { preupgrade_changes echo "" - if [[ $is_airgap -eq 0 ]]; then - echo "Updating Rule Files to the Latest." - update_airgap_rules - fi + echo "Updating Airgap Rule Files to the Latest." + update_airgap_rules # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars if [[ ! "$MINIONID" =~ "_import" ]]; then diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 7c04da825..93ca07ac8 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -22,7 +22,7 @@ so-soc: - sobridge: - ipv4_address: {{ DOCKER.containers['so-soc'].ip }} - binds: - - /nsm/rules:/nsm/rules:rw #Need to tighten this up? + - /nsm/rules:/nsm/rules:rw - /opt/so/conf/strelka:/opt/sensoroni/yara:rw - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index 65091158e..bc7c5cada 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -34,6 +34,12 @@ {% do SOCMERGED.config.server.client.inactiveTools.append('toolPlaybook') %} {% endif %} +{# if system is Airgap, don't autoupdate Yara & Sigma rules #} +{% if pillar.global.airgap %} + {% do SOCMERGED.config.server.modules.elastalertengine.update({'autoUpdateEnabled': false}) %} + {% do SOCMERGED.config.server.modules.strelkaengine.update({'autoUpdateEnabled': false}) %} +{% endif %} + {% set standard_actions = SOCMERGED.config.pop('actions') %} {% if pillar.global.endgamehost != '' %} diff --git a/setup/so-setup b/setup/so-setup index ca1581ef9..e2de39f50 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -687,10 +687,8 @@ if ! [[ -f $install_opt_file ]]; then logCmd "so-minion -o=setup" title "Creating Global SLS" - if [[ $is_airgap ]]; then - # Airgap Rules - airgap_rules - fi + # Airgap Rules + airgap_rules manager_pillar From 59af547838100d006f924ad43e302c7aac736bc6 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 27 Feb 2024 09:49:54 -0500 Subject: [PATCH 070/140] Fix download location --- salt/soc/soc_soc.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index fa488ab7a..e80e98a7e 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -83,7 +83,7 @@ soc: global: True advanced: False autoUpdateEnabled: - description: 'Set to true to enable automatic updates of the Sigma Community Ruleset.' + description: 'Set to true to enable automatic Internet-connected updates of the Sigma Community Ruleset. If this is an Airgap system, this setting will be overridden and set to false.' global: True advanced: True elastic: @@ -146,7 +146,7 @@ soc: advanced: True strelkaengine: autoUpdateEnabled: - description: 'Set to true to enable automatic updates of the Yara ruleset.' + description: 'Set to true to enable automatic Internet-connected updates of the Yara rulesets. If this is an Airgap system, this setting will be overridden and set to false.' global: True advanced: True client: From fcc0f9d14f50019dcad5ffc02035173fe50c6bbe Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 27 Feb 2024 13:20:58 -0500 Subject: [PATCH 071/140] redo classifications --- .../classification/classification.config | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/salt/suricata/classification/classification.config b/salt/suricata/classification/classification.config index e597eb5a1..220736c94 100644 --- a/salt/suricata/classification/classification.config +++ b/salt/suricata/classification/classification.config @@ -1,2 +1,51 @@ -# config classification: shortname,description,priority -# config classification: misc-activity,Misc activity,3 +# +# config classification:shortname,short description,priority +# + +config classification: not-suspicious,Not Suspicious Traffic,3 +config classification: unknown,Unknown Traffic,3 +config classification: bad-unknown,Potentially Bad Traffic, 2 +config classification: attempted-recon,Attempted Information Leak,2 +config classification: successful-recon-limited,Information Leak,2 +config classification: successful-recon-largescale,Large Scale Information Leak,2 +config classification: attempted-dos,Attempted Denial of Service,2 +config classification: successful-dos,Denial of Service,2 +config classification: attempted-user,Attempted User Privilege Gain,1 +config classification: unsuccessful-user,Unsuccessful User Privilege Gain,1 +config classification: successful-user,Successful User Privilege Gain,1 +config classification: attempted-admin,Attempted Administrator Privilege Gain,1 +config classification: successful-admin,Successful Administrator Privilege Gain,1 + +# NEW CLASSIFICATIONS +config classification: rpc-portmap-decode,Decode of an RPC Query,2 +config classification: shellcode-detect,Executable code was detected,1 +config classification: string-detect,A suspicious string was detected,3 +config classification: suspicious-filename-detect,A suspicious filename was detected,2 +config classification: suspicious-login,An attempted login using a suspicious username was detected,2 +config classification: system-call-detect,A system call was detected,2 +config classification: tcp-connection,A TCP connection was detected,4 +config classification: trojan-activity,A Network Trojan was detected, 1 +config classification: unusual-client-port-connection,A client was using an unusual port,2 +config classification: network-scan,Detection of a Network Scan,3 +config classification: denial-of-service,Detection of a Denial of Service Attack,2 +config classification: non-standard-protocol,Detection of a non-standard protocol or event,2 +config classification: protocol-command-decode,Generic Protocol Command Decode,3 +config classification: web-application-activity,access to a potentially vulnerable web application,2 +config classification: web-application-attack,Web Application Attack,1 +config classification: misc-activity,Misc activity,3 +config classification: misc-attack,Misc Attack,2 +config classification: icmp-event,Generic ICMP event,3 +config classification: inappropriate-content,Inappropriate Content was Detected,1 +config classification: policy-violation,Potential Corporate Privacy Violation,1 +config classification: default-login-attempt,Attempt to login by a default username and password,2 + +# Update +config classification: targeted-activity,Targeted Malicious Activity was Detected,1 +config classification: exploit-kit,Exploit Kit Activity Detected,1 +config classification: external-ip-check,Device Retrieving External IP Address Detected,2 +config classification: domain-c2,Domain Observed Used for C2 Detected,1 +config classification: pup-activity,Possibly Unwanted Program Detected,2 +config classification: credential-theft,Successful Credential Theft Detected,1 +config classification: social-engineering,Possible Social Engineering Attempted,2 +config classification: coin-mining,Crypto Currency Mining Activity Detected,2 +config classification: command-and-control,Malware Command and Control Activity Detected,1 From df3943b4651e1bf293e0433d2138dece8ffc829b Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 27 Feb 2024 17:24:27 -0500 Subject: [PATCH 072/140] Daily rollover --- salt/elasticsearch/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 8e28f2e41..39d218564 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -10597,7 +10597,7 @@ elasticsearch: hot: actions: rollover: - max_age: 30d + max_age: 1d max_primary_shard_size: 50gb set_priority: priority: 100 From e2dd0f8cf17950a6b35d66419085adb167a28e34 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 28 Feb 2024 09:39:23 -0500 Subject: [PATCH 073/140] Only update rule files if AG --- salt/manager/tools/sbin/soup | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 655e99f6c..ba8316116 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -935,8 +935,10 @@ main() { preupgrade_changes echo "" - echo "Updating Airgap Rule Files to the Latest." - update_airgap_rules + if [[ $is_airgap -eq 0 ]]; then + echo "Updating Rule Files to the Latest." + update_airgap_rules + fi # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars if [[ ! "$MINIONID" =~ "_import" ]]; then From 53761d4dba278243222822f1a00a1ec9ba033891 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 29 Feb 2024 16:15:26 -0500 Subject: [PATCH 074/140] FIX: EA installers not downloadable from SOC + fix stg logging Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/stig/enabled.sls | 21 ++++++++++++--------- salt/stig/files/sos-oscap.xml | 6 +++--- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/salt/stig/enabled.sls b/salt/stig/enabled.sls index 1f1a064fd..0f347f8bc 100644 --- a/salt/stig/enabled.sls +++ b/salt/stig/enabled.sls @@ -48,15 +48,17 @@ update_stig_profile: {% if not salt['file.file_exists'](OSCAP_OUTPUT_DIR ~ '/pre-oscap-report.html') %} run_initial_scan: - module.run: - - name: openscap.xccdf - - params: 'eval --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/pre-oscap-results.xml --report {{ OSCAP_OUTPUT_DIR }}/pre-oscap-report.html {{ OSCAP_PROFILE_LOCATION }}' + cmd.run: + - name: 'oscap xccdf eval --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/pre-oscap-results.xml --report {{ OSCAP_OUTPUT_DIR }}/pre-oscap-report.html {{ OSCAP_PROFILE_LOCATION }}' + - success_retcodes: + - 2 {% endif %} run_remediate: - module.run: - - name: openscap.xccdf - - params: 'eval --remediate --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/post-oscap-results.xml --report {{ OSCAP_PROFILE_LOCATION }}' + cmd.run: + - name: 'oscap xccdf eval --remediate --profile {{ OSCAP_PROFILE_NAME }} {{ OSCAP_PROFILE_LOCATION }}' + - success_retcodes: + - 2 {# OSCAP rule id: xccdf_org.ssgproject.content_rule_disable_ctrlaltdel_burstaction #} disable_ctrl_alt_del_action: @@ -82,9 +84,10 @@ remove_nullok_from_system_auth_auth: - backup: '.bak' run_post_scan: - module.run: - - name: openscap.xccdf - - params: 'eval --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/post-oscap-results.xml --report {{ OSCAP_OUTPUT_DIR }}/post-oscap-report.html {{ OSCAP_PROFILE_LOCATION }}' + cmd.run: + - name: 'oscap xccdf eval --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/post-oscap-results.xml --report {{ OSCAP_OUTPUT_DIR }}/post-oscap-report.html {{ OSCAP_PROFILE_LOCATION }}' + - success_retcodes: + - 2 {% else %} {{sls}}_no_license_detected: diff --git a/salt/stig/files/sos-oscap.xml b/salt/stig/files/sos-oscap.xml index 3f78af8c0..6c4c93778 100644 --- a/salt/stig/files/sos-oscap.xml +++ b/salt/stig/files/sos-oscap.xml @@ -611,7 +611,7 @@ the release. Additionally, the original security profile has been modified by Se - + @@ -1007,8 +1007,8 @@ the release. Additionally, the original security profile has been modified by Se - - + + From b017157d21a7b32a8163b7593d2279a06d3a776f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 1 Mar 2024 14:04:56 -0500 Subject: [PATCH 075/140] Add antivirus mapping --- salt/soc/files/soc/sigma_so_pipeline.yaml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 533823e6f..37e9f4a3e 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -16,7 +16,25 @@ transformations: src_port: source.port dst_ip: destination.ip.keyword dst_port: destination.port - winlog.event_data.User: user.name + winlog.event_data.User: user.name + # Maps "antivirus" category to Windows Defender logs shipped by Elastic Agent Winlog Integration + # winlog.event_data.threat_name has to be renamed prior to ingestion, it is originally winlog.event_data.Threat Name + - id: antivirus_field-mappings_windows-defender + type: field_name_mapping + mapping: + Signature: winlog.event_data.threat_name + rule_conditions: + - type: logsource + category: antivirus + - id: antivirus_add-fields_windows-defender + type: add_condition + conditions: + winlog.channel: 'Microsoft-Windows-Windows Defender/Operational' + winlog.provider_name: 'Microsoft-Windows-Windows Defender' + event.code: "1116" + rule_conditions: + - type: logsource + category: antivirus - id: hashes_process-creation type: field_name_mapping mapping: From d832158cc52fe7c87d88fe233c38128d425d0a2f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 1 Mar 2024 15:26:02 -0500 Subject: [PATCH 076/140] Drop Hashes field --- salt/soc/files/soc/sigma_so_pipeline.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 37e9f4a3e..d227c3f01 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -35,6 +35,17 @@ transformations: rule_conditions: - type: logsource category: antivirus + # Drops the Hashes field which is specific to Sysmon logs + # Ingested sysmon logs will have the Hashes field mapped to ECS specific fields + - id: hashes_drop_sysmon-specific-field + type: drop_detection_item + field_name_conditions: + - type: include_fields + fields: + - winlog.event_data.Hashes + rule_conditions: + - type: logsource + product: windows - id: hashes_process-creation type: field_name_mapping mapping: @@ -67,4 +78,4 @@ transformations: rule_conditions: - type: logsource product: windows - category: driver_load \ No newline at end of file + category: driver_load From f28f269bb120195d00066d7b7fcf0304906c8e86 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 4 Mar 2024 07:38:32 -0500 Subject: [PATCH 077/140] Fix FIM --- .../tools/sbin_jinja/so-elastic-agent-gen-installers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index 275bc6a11..ff46a3e07 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -46,7 +46,7 @@ do done printf "\n### Stripping out unused components" -find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -maxdepth 1 -regex '.*fleet.*\|.*packet.*\|.*apm.*\|.*audit.*\|.*heart.*\|.*cloud.*' -delete +find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -maxdepth 1 -regex '.*fleet.*\|.*packet.*\|.*apm.*\|.*heart.*\|.*cloud.*' -delete printf "\n### Tarring everything up again" for OS in "${OSARCH[@]}" From 018e099111e86774e51126b42d22253b73ea7495 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 4 Mar 2024 14:53:15 -0500 Subject: [PATCH 078/140] Modify setup --- salt/manager/tools/sbin/so-minion | 39 +++++++++++++++++++------------ 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 4995e1c9d..4a6e5b7c7 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -80,28 +80,27 @@ function getinstallinfo() { } function pcapspace() { - - local NSMSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') - local ROOTSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') - - if [[ "$NSMSIZE" == "null" ]]; then - # Looks like there is no dedicated nsm partition. Using root - local SPACESIZE=$ROOTSIZE + if [[ "$OPERATION" == "setup" ]]; then + local SPACESIZE=$(df -h /nsm | tail -1 | awk '{print $2}') else - local SPACESIZE=$NSMSIZE + + local NSMSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') + local ROOTSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') + + if [[ "$NSMSIZE" == "null" ]]; then + # Looks like there is no dedicated nsm partition. Using root + local SPACESIZE=$ROOTSIZE + else + local SPACESIZE=$NSMSIZE + fi fi local s=$(( $SPACESIZE / 1000000 )) local s1=$(( $s / 2 )) local s2=$(( $s1 / $lb_procs )) - printf '%s\n'\ - "suricata:"\ - " config:"\ - " output:"\ - " pcap-log:"\ - " max-files: $s" >> $PILLARFILE - + MAXPCAPFILES=$s2 + } function testMinion() { @@ -272,6 +271,11 @@ function add_sensor_to_minion() { echo " config:" >> $PILLARFILE echo " af-packet:" >> $PILLARFILE echo " threads: '$CORECOUNT'" >> $PILLARFILE + if [[ $is_pcaplimit ]]; then + echo " output:" >> $PILLARFILE + echo " pcap-log:" >> $PILLARFILE + echo " max-files: '$MAXPCAPFILES'" >> $PILLARFILE + fi echo "pcap:" >> $PILLARFILE echo " enabled: True" >> $PILLARFILE if [[ $is_pcaplimit ]]; then @@ -448,6 +452,7 @@ function updateMine() { function createEVAL() { is_pcaplimit=true + pcapspace add_elasticsearch_to_minion add_sensor_to_minion add_strelka_to_minion @@ -468,6 +473,7 @@ function createEVAL() { function createSTANDALONE() { is_pcaplimit=true + pcapspace add_elasticsearch_to_minion add_logstash_to_minion add_sensor_to_minion @@ -557,6 +563,7 @@ function createIDH() { function createHEAVYNODE() { is_pcaplimit=true + pcapspace add_elasticsearch_to_minion add_elastic_agent_to_minion add_logstash_to_minion @@ -567,6 +574,8 @@ function createHEAVYNODE() { } function createSENSOR() { + is_pcaplimit=true + pcapspace add_sensor_to_minion add_strelka_to_minion add_telegraf_to_minion From fe238755e981fdc3316c48f03c23c9a6a29613ca Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 4 Mar 2024 16:52:51 -0500 Subject: [PATCH 079/140] Fix df --- salt/manager/tools/sbin/so-minion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 4a6e5b7c7..d696e14c6 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -81,7 +81,7 @@ function getinstallinfo() { function pcapspace() { if [[ "$OPERATION" == "setup" ]]; then - local SPACESIZE=$(df -h /nsm | tail -1 | awk '{print $2}') + local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n) else local NSMSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') From b64d61065a7cf910a1c53d28485bfbee21edb4e9 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 5 Mar 2024 09:19:43 -0500 Subject: [PATCH 080/140] Add AWS Cloudfront template --- salt/elasticsearch/defaults.yaml | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 39d218564..0d2dd8a41 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1107,6 +1107,50 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-aws_x_cloudfront_logs: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.cloudfront_logs-*" + template: + settings: + index: + lifecycle: + name: so-logs-aws.cloudfront_logs-logs + number_of_replicas: 0 + composed_of: + - "logs-aws.cloudfront_logs@package" + - "logs-aws.cloudfront_logs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-aws_x_cloudtrail: index_sorting: false index_template: From 1514f1291e2961dedd91354c0593ffa6e0854023 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 5 Mar 2024 09:21:48 -0500 Subject: [PATCH 081/140] Add AWS GuardDuty template --- salt/elasticsearch/defaults.yaml | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 0d2dd8a41..54a65a112 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1371,6 +1371,50 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-aws_x_guardduty: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.guardduty-*" + template: + settings: + index: + lifecycle: + name: so-logs-aws.guardduty-logs + number_of_replicas: 0 + composed_of: + - "logs-aws.guardduty@package" + - "logs-aws.guardduty@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-aws_x_route53_public_logs: index_sorting: false index_template: From d85ac39e2875dac3a58930abd2523f5be7af6ece Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 5 Mar 2024 09:23:17 -0500 Subject: [PATCH 082/140] Add AWS Inspector template --- salt/elasticsearch/defaults.yaml | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 54a65a112..8d31a1acd 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1415,6 +1415,50 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-aws_x_inspector: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.inspector-*" + template: + settings: + index: + lifecycle: + name: so-logs-aws.inspector-logs + number_of_replicas: 0 + composed_of: + - "logs-aws.inspector@package" + - "logs-aws.inspector@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-aws_x_route53_public_logs: index_sorting: false index_template: From d8e8933ea0a035e0628a0bbcc65096a0c0a00b01 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 5 Mar 2024 09:25:41 -0500 Subject: [PATCH 083/140] Add AWS Security Hub template --- salt/elasticsearch/defaults.yaml | 88 ++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 8d31a1acd..2274018b1 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1591,6 +1591,94 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-aws_x_securityhub_findings: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.securityhub_findings-*" + template: + settings: + index: + lifecycle: + name: so-logs-aws.securityhub_findings-logs + number_of_replicas: 0 + composed_of: + - "logs-aws.securityhub_findings@package" + - "logs-aws.securityhub_findings@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-aws_x_securityhub_insights: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.securityhub_insights-*" + template: + settings: + index: + lifecycle: + name: so-logs-aws.securityhub_insights-logs + number_of_replicas: 0 + composed_of: + - "logs-aws.securityhub_insights@package" + - "logs-aws.securityhub_insights@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-aws_x_vpcflow: index_sorting: false index_template: From 2a7e5b096f0320dec1d395bf3fe3c5721582283f Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 5 Mar 2024 09:48:59 -0500 Subject: [PATCH 084/140] Change version for foxtrot --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5a99ed019..7d52aac7f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.60 +2.4.0-foxtrot From bed42208b1856f1e02033ac483a0610fa7adb76e Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 5 Mar 2024 09:49:55 -0500 Subject: [PATCH 085/140] Add journald integration --- salt/elasticfleet/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index e4f54ceb0..7b2d9d6a3 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -65,6 +65,7 @@ elasticfleet: - http_endpoint - httpjson - iis + - journald - juniper - juniper_srx - kafka_log From 08f2b8251b95638ef5611e203bd476f8e259c74c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 5 Mar 2024 09:53:35 -0500 Subject: [PATCH 086/140] add GLOBALS.is_sensor --- salt/vars/globals.map.jinja | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/vars/globals.map.jinja b/salt/vars/globals.map.jinja index 624173217..ed7129678 100644 --- a/salt/vars/globals.map.jinja +++ b/salt/vars/globals.map.jinja @@ -8,6 +8,7 @@ set GLOBALS = { 'hostname': INIT.GRAINS.nodename, 'is_manager': false, + 'is_sensor': false, 'manager': INIT.GRAINS.master, 'minion_id': INIT.GRAINS.id, 'main_interface': INIT.PILLAR.host.mainint, @@ -63,5 +64,8 @@ {% do GLOBALS.update({'is_manager': true}) %} {% endif %} +{% if GLOBALS.role in GLOBALS.sensor_roles %} +{% do GLOBALS.update({'is_sensor': true}) %} +{% endif %} {% do salt['defaults.merge'](GLOBALS, ROLE_GLOBALS, merge_lists=False, in_place=True) %} From 1a58aa61a0409889194fe7f427078c7de5623aff Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 5 Mar 2024 09:54:40 -0500 Subject: [PATCH 087/140] only import pcap and suricata if sensor --- salt/sensoroni/files/sensoroni.json | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 97c91f0b3..93708440a 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -1,7 +1,5 @@ -{%- from 'vars/globals.map.jinja' import GLOBALS %} -{%- from 'sensoroni/map.jinja' import SENSORONIMERGED %} -{%- from 'pcap/config.map.jinja' import PCAPMERGED %} -{%- from 'suricata/map.jinja' import SURICATAMERGED %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{%- from 'sensoroni/map.jinja' import SENSORONIMERGED -%} { "logFilename": "/opt/sensoroni/logs/sensoroni.log", "logLevel":"info", @@ -24,24 +22,28 @@ "importer": {}, "statickeyauth": { "apiKey": "{{ GLOBALS.sensoroni_key }}" -{#- if PCAPMERGED.enabled is true then we know that steno is the pcap engine #} -{#- if it is false, then user has steno disabled in ui or has selected suricata for pcap engine #} -{%- if PCAPMERGED.enabled %} +{% if GLOBALS.is_sensor %} +{% from 'pcap/config.map.jinja' import PCAPMERGED %} +{% from 'suricata/map.jinja' import SURICATAMERGED %} +{# if PCAPMERGED.enabled is true then we know that steno is the pcap engine #} +{# if it is false, then user has steno disabled in ui or has selected suricata for pcap engine #} +{%- if PCAPMERGED.enabled %} }, "stenoquery": { "executablePath": "/opt/sensoroni/scripts/stenoquery.sh", "pcapInputPath": "/nsm/pcap", "pcapOutputPath": "/nsm/pcapout" } -{%- elif GLOBALS.pcap_engine == "SURICATA" and SURICATAMERGED.enabled %} +{%- elif GLOBALS.pcap_engine == "SURICATA" and SURICATAMERGED.enabled %} }, "suriquery": { "executablePath": "/opt/sensoroni/scripts/suriquery.sh", "pcapInputPath": "/nsm/suripcap", "pcapOutputPath": "/nsm/pcapout" } -{%- else %} +{%- else %} } +{% endif %} {%- endif %} } } From c0d19e11b9bad6ee1ebb93088f42f3e44b0bc13c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 5 Mar 2024 10:07:32 -0500 Subject: [PATCH 088/140] fix } placement --- salt/sensoroni/files/sensoroni.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 93708440a..5090967ef 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -41,9 +41,9 @@ "pcapInputPath": "/nsm/suripcap", "pcapOutputPath": "/nsm/pcapout" } -{%- else %} +{% endif %} +{%- else %} } -{% endif %} {%- endif %} } } From b9ebe6c40b2545bfb9c0d18e75d2be67c5f3d9f2 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 5 Mar 2024 12:58:34 -0500 Subject: [PATCH 089/140] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7d52aac7f..5a99ed019 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-foxtrot +2.4.60 From 6eb608c3f53f2a9b6743d02eebe080c469343995 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 5 Mar 2024 15:05:03 -0500 Subject: [PATCH 090/140] Update so-minion --- salt/manager/tools/sbin/so-minion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index d696e14c6..82c19e39b 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -96,7 +96,7 @@ function pcapspace() { fi local s=$(( $SPACESIZE / 1000000 )) - local s1=$(( $s / 2 )) + local s1=$(( $s / 4 )) local s2=$(( $s1 / $lb_procs )) MAXPCAPFILES=$s2 From a686d46322ed335c8a7fd4220843e823511f2769 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 5 Mar 2024 15:09:02 -0500 Subject: [PATCH 091/140] Update so-minion --- salt/manager/tools/sbin/so-minion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 82c19e39b..7e33533b4 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -280,7 +280,7 @@ function add_sensor_to_minion() { echo " enabled: True" >> $PILLARFILE if [[ $is_pcaplimit ]]; then echo " config:" >> $PILLARFILE - echo " diskfreepercentage: 60" >> $PILLARFILE + echo " diskfreepercentage: 75" >> $PILLARFILE pcapspace fi echo " " >> $PILLARFILE From 4b5f00cef4d13a21ac8ff635a8ec625151e33e07 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 5 Mar 2024 16:42:20 -0500 Subject: [PATCH 092/140] fix oinkcodes with leading zeros --- salt/idstools/soc_idstools.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml index f8ec3b8b6..3e3a68117 100644 --- a/salt/idstools/soc_idstools.yaml +++ b/salt/idstools/soc_idstools.yaml @@ -6,6 +6,7 @@ idstools: description: Enter your registration code or oinkcode for paid NIDS rulesets. title: Registration Code global: True + forcedType: string helpLink: rules.html ruleset: description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. WARNING! Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.' From 5687fdcf578eb44aa11f72646a11c284ea57838e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 5 Mar 2024 17:46:43 -0500 Subject: [PATCH 093/140] fix pcapspace function --- salt/manager/tools/sbin/so-minion | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index d696e14c6..54587774f 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -84,8 +84,8 @@ function pcapspace() { local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n) else - local NSMSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') - local ROOTSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') + local NSMSIZE=$(salt "$MINION_ID" disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') + local ROOTSIZE=$(salt "$MINION_ID" disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') if [[ "$NSMSIZE" == "null" ]]; then # Looks like there is no dedicated nsm partition. Using root @@ -97,7 +97,7 @@ function pcapspace() { local s=$(( $SPACESIZE / 1000000 )) local s1=$(( $s / 2 )) - local s2=$(( $s1 / $lb_procs )) + local s2=$(( $s1 / $CORECOUNT )) MAXPCAPFILES=$s2 From eaef076eba34104b5495aef002b8bab3e1b8422b Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 5 Mar 2024 17:52:24 -0500 Subject: [PATCH 094/140] Update so-minion --- salt/manager/tools/sbin/so-minion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 6f037d344..ab05fafcc 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -96,8 +96,8 @@ function pcapspace() { fi local s=$(( $SPACESIZE / 1000000 )) - local s2=$(( $s1 / $CORECOUNT )) local s1=$(( $s / 4 )) + local s2=$(( $s1 / $CORECOUNT )) MAXPCAPFILES=$s2 From 1b47537a3f2d808e8cc9176fbfe636edadea6c7f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 6 Mar 2024 07:16:50 -0500 Subject: [PATCH 095/140] Add Exclusion toggle --- salt/soc/defaults.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d5ecbe71f..60f2ee613 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1201,6 +1201,9 @@ soc: - name: caseExcludeToggle filter: 'NOT _index:"*:so-case*"' enabled: true + - name: detectionsExcludeToggle + filter: 'NOT _index:"*:so-detection*"' + enabled: true - name: socExcludeToggle filter: 'NOT event.module:"soc"' enabled: true @@ -1471,6 +1474,9 @@ soc: - name: caseExcludeToggle filter: 'NOT _index:"*:so-case*"' enabled: true + - name: detectionsExcludeToggle + filter: 'NOT _index:"*:so-detection*"' + enabled: true - name: socExcludeToggle filter: 'NOT event.module:"soc"' enabled: true From 12653eec8c4565e9d2538c803fc9a8e53edfb1c6 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 6 Mar 2024 08:14:33 -0500 Subject: [PATCH 096/140] add new pcap annotations --- salt/sensoroni/defaults.yaml | 118 ++++++++++++++-------------- salt/sensoroni/files/sensoroni.json | 3 +- salt/sensoroni/soc_sensoroni.yaml | 5 ++ salt/soc/defaults.yaml | 1 + salt/soc/soc_soc.yaml | 3 + 5 files changed, 71 insertions(+), 59 deletions(-) diff --git a/salt/sensoroni/defaults.yaml b/salt/sensoroni/defaults.yaml index 4ebd666a9..7777985dd 100644 --- a/salt/sensoroni/defaults.yaml +++ b/salt/sensoroni/defaults.yaml @@ -1,58 +1,60 @@ -sensoroni: - enabled: False - config: - analyze: - enabled: False - timeout_ms: 900000 - parallel_limit: 5 - node_checkin_interval_ms: 10000 - sensoronikey: - soc_host: - analyzers: - echotrail: - base_url: https://api.echotrail.io/insights/ - api_key: - elasticsearch: - base_url: - auth_user: - auth_pwd: - num_results: 10 - api_key: - index: _all - time_delta_minutes: 14400 - timestamp_field_name: '@timestamp' - map: {} - cert_path: - emailrep: - base_url: https://emailrep.io/ - api_key: - greynoise: - base_url: https://api.greynoise.io/ - api_key: - api_version: community - localfile: - file_path: [] - otx: - base_url: https://otx.alienvault.com/api/v1/ - api_key: - pulsedive: - base_url: https://pulsedive.com/api/ - api_key: - spamhaus: - lookup_host: zen.spamhaus.org - nameservers: [] - sublime_platform: - base_url: https://api.platform.sublimesecurity.com - api_key: - live_flow: False - mailbox_email_address: - message_source_id: - urlscan: - base_url: https://urlscan.io/api/v1/ - api_key: - enabled: False - visibility: public - timeout: 180 - virustotal: - base_url: https://www.virustotal.com/api/v3/search?query= - api_key: +sensoroni: + enabled: False + config: + analyze: + enabled: False + timeout_ms: 900000 + parallel_limit: 5 + node_checkin_interval_ms: 10000 + sensoronikey: + soc_host: + suripcap: + pcapMaxCount: 999999 + analyzers: + echotrail: + base_url: https://api.echotrail.io/insights/ + api_key: + elasticsearch: + base_url: + auth_user: + auth_pwd: + num_results: 10 + api_key: + index: _all + time_delta_minutes: 14400 + timestamp_field_name: '@timestamp' + map: {} + cert_path: + emailrep: + base_url: https://emailrep.io/ + api_key: + greynoise: + base_url: https://api.greynoise.io/ + api_key: + api_version: community + localfile: + file_path: [] + otx: + base_url: https://otx.alienvault.com/api/v1/ + api_key: + pulsedive: + base_url: https://pulsedive.com/api/ + api_key: + spamhaus: + lookup_host: zen.spamhaus.org + nameservers: [] + sublime_platform: + base_url: https://api.platform.sublimesecurity.com + api_key: + live_flow: False + mailbox_email_address: + message_source_id: + urlscan: + base_url: https://urlscan.io/api/v1/ + api_key: + enabled: False + visibility: public + timeout: 180 + virustotal: + base_url: https://www.virustotal.com/api/v3/search?query= + api_key: diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 5090967ef..eb9c1131d 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -39,7 +39,8 @@ "suriquery": { "executablePath": "/opt/sensoroni/scripts/suriquery.sh", "pcapInputPath": "/nsm/suripcap", - "pcapOutputPath": "/nsm/pcapout" + "pcapOutputPath": "/nsm/pcapout", + "pcapMaxCount": {{ SENSORONIMERGED.config.suripcap.pcapMaxCount }} } {% endif %} {%- else %} diff --git a/salt/sensoroni/soc_sensoroni.yaml b/salt/sensoroni/soc_sensoroni.yaml index f7a10c6f7..7b8495dc5 100644 --- a/salt/sensoroni/soc_sensoroni.yaml +++ b/salt/sensoroni/soc_sensoroni.yaml @@ -37,6 +37,11 @@ sensoroni: helpLink: grid.html global: True advanced: True + suripcap: + pcapMaxCount: + description: The maximum number of PCAP packets to extract from eligible PCAP files, for PCAP jobs. If there are issues fetching excessively large packet streams consider lowering this value to reduce the number of collected packets returned to the user interface. + helpLink: sensoroni.html + advanced: True analyzers: echotrail: api_key: diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d5ecbe71f..abcd12308 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1103,6 +1103,7 @@ soc: esSearchOffsetMs: 1800000 maxLogLength: 1024 asyncThreshold: 10 + lookupTunnelParent: true influxdb: hostUrl: token: diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index e80e98a7e..645a4c8a9 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -126,6 +126,9 @@ soc: description: Maximum number of events that can be acknowledged synchronously. When acknowledging large numbers of events, where the count exceeds this value, the acknowledge update will be performed in the background, as it can take several minutes to complete. global: True advanced: True + lookupTunnelParent: + description: When true, if a pivoted event appears to be encapsulated, such as in a VxLan packet, then SOC will pivot to the VxLan packet stream. This can be useful if the PCAP parser is unable to locate the encapsulated packets. However, if the parser is written in a way that it can find a given filter even if its encapsulated, this is best left to false, as the analyst will see the intended packet data rather than the wrapping packet data. + global: True sostatus: refreshIntervalMs: description: Duration (in milliseconds) between refreshes of the grid status. Shortening this duration may not have expected results, as the backend systems feeding this sostatus data will continue their updates as scheduled. From 0f12297f5019e91def5af525ffcd113108dc9e29 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 6 Mar 2024 08:19:42 -0500 Subject: [PATCH 097/140] add new pcap annotations --- salt/soc/soc_soc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 645a4c8a9..799d9af4e 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -127,7 +127,7 @@ soc: global: True advanced: True lookupTunnelParent: - description: When true, if a pivoted event appears to be encapsulated, such as in a VxLan packet, then SOC will pivot to the VxLan packet stream. This can be useful if the PCAP parser is unable to locate the encapsulated packets. However, if the parser is written in a way that it can find a given filter even if its encapsulated, this is best left to false, as the analyst will see the intended packet data rather than the wrapping packet data. + description: When true, if a pivoted event appears to be encapsulated, such as in a VXLAN packet, then SOC will pivot to the VXLAN packet stream. When false, SOC will attempt to pivot to the encapsulated packet stream itself, but at the risk that it may be unable to locate it in the stored PCAP data. global: True sostatus: refreshIntervalMs: From f58c104d899bf99fa1d3b566797eb2447a909917 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 6 Mar 2024 09:51:56 -0500 Subject: [PATCH 098/140] Update so-minion --- salt/manager/tools/sbin/so-minion | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index ab05fafcc..09708707f 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -81,6 +81,8 @@ function getinstallinfo() { function pcapspace() { if [[ "$OPERATION" == "setup" ]]; then + # Use 25% for PCAP + PCAP_PERCENTAGE=1 local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n) else @@ -96,10 +98,9 @@ function pcapspace() { fi local s=$(( $SPACESIZE / 1000000 )) - local s1=$(( $s / 4 )) - local s2=$(( $s1 / $CORECOUNT )) + local s1=$(( $s / 4 * $PCAP_PERCENTAGE )) - MAXPCAPFILES=$s2 + MAX_PCAP_SPACE=$s1 } @@ -271,17 +272,12 @@ function add_sensor_to_minion() { echo " config:" >> $PILLARFILE echo " af-packet:" >> $PILLARFILE echo " threads: '$CORECOUNT'" >> $PILLARFILE - if [[ $is_pcaplimit ]]; then - echo " output:" >> $PILLARFILE - echo " pcap-log:" >> $PILLARFILE - echo " max-files: '$MAXPCAPFILES'" >> $PILLARFILE - fi echo "pcap:" >> $PILLARFILE echo " enabled: True" >> $PILLARFILE if [[ $is_pcaplimit ]]; then echo " config:" >> $PILLARFILE echo " diskfreepercentage: 75" >> $PILLARFILE - pcapspace + echo " suripcapmaxsize: $MAX_PCAP_SPACE" >> $PILLARFILE fi echo " " >> $PILLARFILE } @@ -563,6 +559,7 @@ function createIDH() { function createHEAVYNODE() { is_pcaplimit=true + PCAP_PERCENTAGE=1 pcapspace add_elasticsearch_to_minion add_elastic_agent_to_minion @@ -575,6 +572,7 @@ function createHEAVYNODE() { function createSENSOR() { is_pcaplimit=true + PCAP_PERCENTAGE=3 pcapspace add_sensor_to_minion add_strelka_to_minion From a63fca727ce144a90d6fd01a82707ecfcf32a39f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 6 Mar 2024 10:02:06 -0500 Subject: [PATCH 099/140] Update soc_suricata.yaml --- salt/suricata/soc_suricata.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 4f9a80d86..fbd6e84ee 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -188,17 +188,11 @@ suricata: advanced: True readonly: True helpLink: suricata.html - limit: - description: File size limit per thread. To determine max PCAP size multiple threads x max-files x limit. - helpLink: suricata.html mode: description: Suricata PCAP mode. Currently only multi is supported. advanced: True readonly: True helpLink: suricata.html - max-files: - description: Max PCAP files per thread. To determine max PCAP size multiple threads x max-files x limit. - helpLink: suricata.html use-stream-depth: description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. advanced: True From 4dfa1a5626387ab70c389565fa74a542cc44949b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 6 Mar 2024 10:35:10 -0500 Subject: [PATCH 101/140] Move Suricata around --- salt/suricata/defaults.yaml | 5 +++-- salt/suricata/soc_suricata.yaml | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index eb2c181e3..42af3fc55 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -1,5 +1,8 @@ suricata: enabled: False + pcap: + filesize: 1000mb + maxsize: 25 config: threading: set-cpu-affinity: "no" @@ -132,9 +135,7 @@ suricata: lz4-checksum: "no" lz4-level: 8 filename: "%n/so-pcap.%t" - limit: "1000mb" mode: "multi" - max-files: 10 use-stream-depth: "no" conditional: "all" dir: "/nsm/suripcap" diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index fbd6e84ee..88b460af8 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -19,6 +19,14 @@ suricata: multiline: True title: Classifications helpLink: suricata.html + pcap: + filesize: + description: Max file size for individual PCAP files written by Suricata. Increasing this number could improve write performance at the expense of pcap retrieval times. + advanced: True + helplink: suricata.html + maxsize: + description: Size in GB for total usage size of PCAP on disk. + helplink: suricata.html config: af-packet: interface: From 167aff24f61b8bbcaced10e05f8dfde3dd4acf0a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 6 Mar 2024 11:03:52 -0500 Subject: [PATCH 102/140] detections annotations --- salt/soc/defaults.yaml | 1 + salt/soc/soc_soc.yaml | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index abcd12308..ad1f8bb5f 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1147,6 +1147,7 @@ soc: tipTimeoutMs: 6000 cacheExpirationMs: 300000 casesEnabled: true + detectionsEnabled: false inactiveTools: ['toolUnused'] tools: - name: toolKibana diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 799d9af4e..08a29766d 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -79,11 +79,11 @@ soc: modules: elastalertengine: sigmaRulePackages: - description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). WARNING! Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.' + description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). WARNING! Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone. (future use, not yet complete)' global: True advanced: False autoUpdateEnabled: - description: 'Set to true to enable automatic Internet-connected updates of the Sigma Community Ruleset. If this is an Airgap system, this setting will be overridden and set to false.' + description: 'Set to true to enable automatic Internet-connected updates of the Sigma Community Ruleset. If this is an Airgap system, this setting will be overridden and set to false. (future use, not yet complete)' global: True advanced: True elastic: @@ -149,7 +149,7 @@ soc: advanced: True strelkaengine: autoUpdateEnabled: - description: 'Set to true to enable automatic Internet-connected updates of the Yara rulesets. If this is an Airgap system, this setting will be overridden and set to false.' + description: 'Set to true to enable automatic Internet-connected updates of the Yara rulesets. If this is an Airgap system, this setting will be overridden and set to false. (future use, not yet complete)' global: True advanced: True client: @@ -174,6 +174,9 @@ soc: casesEnabled: description: Set to true to enable case management in SOC. global: True + detectionsEnabled: + description: Set to true to enable the Detections module in SOC. (future use, not yet complete) + global: True inactiveTools: description: List of external tools to remove from the SOC UI. global: True From ad120934295999845436cdcb303ac3ca51c621e6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 6 Mar 2024 11:05:06 -0500 Subject: [PATCH 103/140] Fix percent calc --- salt/manager/tools/sbin/so-minion | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 09708707f..a3d8230b5 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -83,6 +83,7 @@ function pcapspace() { if [[ "$OPERATION" == "setup" ]]; then # Use 25% for PCAP PCAP_PERCENTAGE=1 + DFREEPERCENT=75 local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n) else @@ -269,6 +270,10 @@ function add_sensor_to_minion() { echo " lb_procs: '$CORECOUNT'" >> $PILLARFILE echo "suricata:" >> $PILLARFILE echo " enabled: True " >> $PILLARFILE + if [[ $is_pcaplimit ]]; then + echo " pcap:" >> $PILLARFILE + echo " maxsize: $MAX_PCAP_SPACE" >> $PILLARFILE + fi echo " config:" >> $PILLARFILE echo " af-packet:" >> $PILLARFILE echo " threads: '$CORECOUNT'" >> $PILLARFILE @@ -276,8 +281,7 @@ function add_sensor_to_minion() { echo " enabled: True" >> $PILLARFILE if [[ $is_pcaplimit ]]; then echo " config:" >> $PILLARFILE - echo " diskfreepercentage: 75" >> $PILLARFILE - echo " suripcapmaxsize: $MAX_PCAP_SPACE" >> $PILLARFILE + echo " diskfreepercentage: $DFREEPERCENT" >> $PILLARFILE fi echo " " >> $PILLARFILE } @@ -560,6 +564,7 @@ function createIDH() { function createHEAVYNODE() { is_pcaplimit=true PCAP_PERCENTAGE=1 + DFREEPERCENT=75 pcapspace add_elasticsearch_to_minion add_elastic_agent_to_minion @@ -572,6 +577,7 @@ function createHEAVYNODE() { function createSENSOR() { is_pcaplimit=true + DFREEPERCENT=10 PCAP_PERCENTAGE=3 pcapspace add_sensor_to_minion From 1cbac11fae137961e67155122856a816989c8a60 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 6 Mar 2024 11:08:03 -0500 Subject: [PATCH 104/140] detections annotations --- salt/soc/soc_soc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 08a29766d..cb939f758 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -34,7 +34,7 @@ soc: helpLink: soc-customization.html sigma_final_pipeline__yaml: title: Final Sigma Pipeline - description: Final Processing Pipeline for Sigma Rules + description: Final Processing Pipeline for Sigma Rules (future use, not yet complete) syntax: yaml file: True global: True From 9a413a2e3189aa24bc654c9ebe388e16153b898f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 6 Mar 2024 12:42:22 -0500 Subject: [PATCH 105/140] Fix location of repo --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index ef1df4a71..4aae0f5bd 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1811,8 +1811,8 @@ repo_sync_local() { info "Adding Repo Download Configuration" mkdir -p /nsm/repo mkdir -p /opt/so/conf/reposync/cache - echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9.3" > /opt/so/conf/reposync/mirror.txt - echo "https://repo-alt.securityonion.net/prod/2.4/oracle/9.3" >> /opt/so/conf/reposync/mirror.txt + echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9" > /opt/so/conf/reposync/mirror.txt + echo "https://repo-alt.securityonion.net/prod/2.4/oracle/9" >> /opt/so/conf/reposync/mirror.txt echo "[main]" > /opt/so/conf/reposync/repodownload.conf echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf echo "installonly_limit=3" >> /opt/so/conf/reposync/repodownload.conf From 7f1e786e3d1e75297ecb7bda12a63a60d6521904 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 6 Mar 2024 12:56:09 -0500 Subject: [PATCH 106/140] Consolidate PCAP settings --- salt/suricata/defaults.yaml | 16 +++---- salt/suricata/soc_suricata.yaml | 84 ++++++++++++++++----------------- 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index 42af3fc55..0252d3a81 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -3,6 +3,14 @@ suricata: pcap: filesize: 1000mb maxsize: 25 + compression: "none" + lz4-checksum: "no" + lz4-level: 8 + filename: "%n/so-pcap.%t" + mode: "multi" + use-stream-depth: "no" + conditional: "all" + dir: "/nsm/suripcap" config: threading: set-cpu-affinity: "no" @@ -131,14 +139,6 @@ suricata: enabled: "no" pcap-log: enabled: "no" - compression: "none" - lz4-checksum: "no" - lz4-level: 8 - filename: "%n/so-pcap.%t" - mode: "multi" - use-stream-depth: "no" - conditional: "all" - dir: "/nsm/suripcap" alert-debug: enabled: "no" alert-prelude: diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 88b460af8..da7586e97 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -27,6 +27,47 @@ suricata: maxsize: description: Size in GB for total usage size of PCAP on disk. helplink: suricata.html + compression: + description: Enable compression of Suricata PCAP. Currently unsupported + advanced: True + readonly: True + helpLink: suricata.html + lz4-checksum: + description: Enable PCAP lz4 checksum. Currently unsupported + advanced: True + readonly: True + helpLink: suricata.html + lz4-level: + description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported + advanced: True + readonly: True + helpLink: suricata.html + filename: + description: Filename output for Suricata PCAP. + advanced: True + readonly: True + helpLink: suricata.html + mode: + description: Suricata PCAP mode. Currently only multi is supported. + advanced: True + readonly: True + helpLink: suricata.html + use-stream-depth: + description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. + advanced: True + regex: ^(yes|no)$ + regexFailureMessage: You must enter either yes or no. + helpLink: suricata.html + conditional: + description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules. + regex: ^(all|alert|tag)$ + regexFailureMessage: You must enter either all, alert or tag. + helpLink: suricata.html + dir: + description: Parent directory to store PCAP. + advanced: True + readonly: True + helpLink: suricata.html config: af-packet: interface: @@ -175,48 +216,7 @@ suricata: description: This value is ignored by SO. pcapengine in globals takes precidence. readonly: True helpLink: suricata.html - advanced: True - compression: - description: Enable compression of Suricata PCAP. Currently unsupported - advanced: True - readonly: True - helpLink: suricata.html - lz4-checksum: - description: Enable PCAP lz4 checksum. Currently unsupported - advanced: True - readonly: True - helpLink: suricata.html - lz4-level: - description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported - advanced: True - readonly: True - helpLink: suricata.html - filename: - description: Filename output for Suricata PCAP. - advanced: True - readonly: True - helpLink: suricata.html - mode: - description: Suricata PCAP mode. Currently only multi is supported. - advanced: True - readonly: True - helpLink: suricata.html - use-stream-depth: - description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. - advanced: True - regex: ^(yes|no)$ - regexFailureMessage: You must enter either yes or no. - helpLink: suricata.html - conditional: - description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules. - regex: ^(all|alert|tag)$ - regexFailureMessage: You must enter either all, alert or tag. - helpLink: suricata.html - dir: - description: Parent directory to store PCAP. - advanced: True - readonly: True - helpLink: suricata.html + advanced: True asn1-max-frames: description: Maximum nuber of asn1 frames to decode. helpLink: suricata.html From cf232534ca9a1609da8946adc24dd5f7116cfc24 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 6 Mar 2024 14:42:07 -0500 Subject: [PATCH 107/140] move suricata.pcap to suricata.config.outputs.pcap-log --- salt/suricata/map.jinja | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 5f6e913f5..d8ce271fe 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -63,6 +63,18 @@ {# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #} {% if GLOBALS.pcap_engine == "SURICATA" %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %} +{# move the items in suricata.pcap into suricata.config.outputs.pcap-log. these items were placed under suricata.config for ease of access in SOC #} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'compression': SURICATAMERGED.pcap.compression}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-checksum': SURICATAMERGED.pcap['lz4-checksum']}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-level': SURICATAMERGED.pcap['lz4-level']}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'filename': SURICATAMERGED.pcap.filename}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'limit': SURICATAMERGED.pcap.filesize}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'mode': SURICATAMERGED.pcap.mode}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-stream-depth': SURICATAMERGED.pcap['max-stream-depth']}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %} +{% set maxfiles = (SURICATAMERGED.pcap.maxsize / SURICATAMERGED.pcap.filesize) | round | int %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %} {% endif %} {# outputs is a list but we convert to dict in defaults to work with ui #} From 583227290f1fedc68ffb9abf14b2b92c8677171d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 6 Mar 2024 15:18:22 -0500 Subject: [PATCH 108/140] fix max-files calc --- salt/suricata/map.jinja | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index d8ce271fe..77cdbe7c5 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -70,10 +70,11 @@ {% do SURICATAMERGED.config.outputs['pcap-log'].update({'filename': SURICATAMERGED.pcap.filename}) %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'limit': SURICATAMERGED.pcap.filesize}) %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'mode': SURICATAMERGED.pcap.mode}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-stream-depth': SURICATAMERGED.pcap['max-stream-depth']}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'use-stream-depth': SURICATAMERGED.pcap['use-stream-depth']}) %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %} -{% set maxfiles = (SURICATAMERGED.pcap.maxsize / SURICATAMERGED.pcap.filesize) | round | int %} +{# multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #} +{% set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / SURICATAMERGED.pcap.filesize[:-2] | int) | round | int %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %} {% endif %} From 17a75d5bd251e758706e53380f29d89930eed997 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 6 Mar 2024 17:19:01 -0500 Subject: [PATCH 109/140] Run stig post remediate scan against default ol9 scap-security-guide. Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/stig/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/stig/enabled.sls b/salt/stig/enabled.sls index 0f347f8bc..c35c91a55 100644 --- a/salt/stig/enabled.sls +++ b/salt/stig/enabled.sls @@ -85,7 +85,7 @@ remove_nullok_from_system_auth_auth: run_post_scan: cmd.run: - - name: 'oscap xccdf eval --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/post-oscap-results.xml --report {{ OSCAP_OUTPUT_DIR }}/post-oscap-report.html {{ OSCAP_PROFILE_LOCATION }}' + - name: 'oscap xccdf eval --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/post-oscap-results.xml --report {{ OSCAP_OUTPUT_DIR }}/post-oscap-report.html /usr/share/xml/scap/ssg/content/ssg-ol9-ds.xml' - success_retcodes: - 2 From 70f3ce0536b4c4aa1c7b24bbea28968ec5fb7c28 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 6 Mar 2024 17:32:06 -0500 Subject: [PATCH 110/140] change how maxfiles is calculated --- salt/suricata/map.jinja | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 77cdbe7c5..6ba3c3b73 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -8,6 +8,24 @@ {% set surimeta_evelog_index = [] %} {% set surimeta_filestore_index = [] %} +{# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #} +{% if GLOBALS.pcap_engine == "SURICATA" %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %} +{# move the items in suricata.pcap into suricata.config.outputs.pcap-log. these items were placed under suricata.config for ease of access in SOC #} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'compression': SURICATAMERGED.pcap.compression}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-checksum': SURICATAMERGED.pcap['lz4-checksum']}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-level': SURICATAMERGED.pcap['lz4-level']}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'filename': SURICATAMERGED.pcap.filename}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'limit': SURICATAMERGED.pcap.filesize}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'mode': SURICATAMERGED.pcap.mode}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'use-stream-depth': SURICATAMERGED.pcap['use-stream-depth']}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %} +{# multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #} +{% set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round | int %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %} +{% endif %} + {# suricata.config.af-packet has to be rewritten here since we cant display '- interface' in the ui #} {# we are limited to only one iterface #} {% load_yaml as afpacket %} @@ -60,24 +78,6 @@ {% do SURICATAMERGED.config.outputs['file-store'].update({'enabled':suricata_mdengine.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %} {% endif %} -{# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #} -{% if GLOBALS.pcap_engine == "SURICATA" %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %} -{# move the items in suricata.pcap into suricata.config.outputs.pcap-log. these items were placed under suricata.config for ease of access in SOC #} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'compression': SURICATAMERGED.pcap.compression}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-checksum': SURICATAMERGED.pcap['lz4-checksum']}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-level': SURICATAMERGED.pcap['lz4-level']}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'filename': SURICATAMERGED.pcap.filename}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'limit': SURICATAMERGED.pcap.filesize}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'mode': SURICATAMERGED.pcap.mode}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'use-stream-depth': SURICATAMERGED.pcap['use-stream-depth']}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %} -{# multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #} -{% set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / SURICATAMERGED.pcap.filesize[:-2] | int) | round | int %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %} -{% endif %} - {# outputs is a list but we convert to dict in defaults to work with ui #} {# below they are converted back to lists #} {% load_yaml as outputs %} From 005930f7fd70577ced8bec4f25650ec42fb1eccd Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 7 Mar 2024 15:41:23 +0000 Subject: [PATCH 111/140] Add error.message mapping for system.syslog --- .../logs-system.syslog@custom.json | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json new file mode 100644 index 000000000..0123fb956 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json @@ -0,0 +1,22 @@ +{ + "template": { + "mappings": { + "properties": { + "error": { + "properties": { + "message": { + "type": "match_only_text" + } + } + } + } + } + }, + "_meta": { + "package": { + "name": "system" + }, + "managed_by": "fleet", + "managed": true + } +} From fffef9b621d9afff1d6fcdae17c8a0fab76aadc5 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 7 Mar 2024 12:31:51 -0500 Subject: [PATCH 112/140] gracefully handle status check failure on ubuntu --- salt/common/tools/sbin/so-common-status-check | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-common-status-check b/salt/common/tools/sbin/so-common-status-check index d713ba6c6..625e0f199 100644 --- a/salt/common/tools/sbin/so-common-status-check +++ b/salt/common/tools/sbin/so-common-status-check @@ -47,10 +47,14 @@ def check_for_fps(): fps = 1 except FileNotFoundError: fn = '/proc/sys/crypto/' + feat_full + '_enabled' - with open(fn, 'r') as f: - contents = f.read() - if '1' in contents: - fps = 1 + try: + with open(fn, 'r') as f: + contents = f.read() + if '1' in contents: + fps = 1 + except: + # Unknown, so assume 0 + fps = 0 with open('/opt/so/log/sostatus/lks_enabled', 'w') as f: f.write(str(fps)) From 40574982e4afbf458a60d77267f1b2fde460fee9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 7 Mar 2024 14:25:43 -0500 Subject: [PATCH 113/140] unswap files --- salt/common/tools/sbin/so-common-status-check | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common-status-check b/salt/common/tools/sbin/so-common-status-check index 625e0f199..4aa981123 100644 --- a/salt/common/tools/sbin/so-common-status-check +++ b/salt/common/tools/sbin/so-common-status-check @@ -56,7 +56,7 @@ def check_for_fps(): # Unknown, so assume 0 fps = 0 - with open('/opt/so/log/sostatus/lks_enabled', 'w') as f: + with open('/opt/so/log/sostatus/fps_enabled', 'w') as f: f.write(str(fps)) def check_for_lks(): @@ -80,7 +80,7 @@ def check_for_lks(): lks = 1 if lks: break - with open('/opt/so/log/sostatus/fps_enabled', 'w') as f: + with open('/opt/so/log/sostatus/lks_enabled', 'w') as f: f.write(str(lks)) def fail(msg): From 06257b9c4a0c046ca4d44ca9b1a301f799a92ec8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 7 Mar 2024 14:32:46 -0500 Subject: [PATCH 114/140] Update so-minion --- salt/manager/tools/sbin/so-minion | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index a3d8230b5..cb4e40ade 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -83,7 +83,7 @@ function pcapspace() { if [[ "$OPERATION" == "setup" ]]; then # Use 25% for PCAP PCAP_PERCENTAGE=1 - DFREEPERCENT=75 + DFREEPERCENT=21 local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n) else @@ -564,7 +564,7 @@ function createIDH() { function createHEAVYNODE() { is_pcaplimit=true PCAP_PERCENTAGE=1 - DFREEPERCENT=75 + DFREEPERCENT=21 pcapspace add_elasticsearch_to_minion add_elastic_agent_to_minion From 3eb6fe2df97b76059ec9876f082061773ec4c71b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 8 Mar 2024 09:52:12 -0500 Subject: [PATCH 115/140] allow managersearch to receiver redis and 5644 --- salt/firewall/defaults.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 75df49b25..75a70828e 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -1295,6 +1295,10 @@ firewall: portgroups: - redis - beats_5644 + managersearch: + portgroups: + - redis + - beats_5644 self: portgroups: - redis From 4e329359919367869ff8a0d3032e8823386863af Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 8 Mar 2024 16:24:37 +0000 Subject: [PATCH 116/140] Add Strelka config back --- salt/manager/init.sls | 45 +++++++++++++++++++++++++++++++++ salt/strelka/backend/config.sls | 10 ++++++++ 2 files changed, 55 insertions(+) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 51590a6ec..56e72c279 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -103,6 +103,51 @@ rules_dir: - group: socore - makedirs: True +{% if STRELKAMERGED.rules.enabled %} + strelkarepos: + file.managed: + - name: /opt/so/conf/strelka/repos.txt + - source: salt://strelka/rules/repos.txt.jinja + - template: jinja + - defaults: + STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} + - makedirs: True + strelka-yara-update: + {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} + cron.present: + {% else %} + cron.absent: + {% endif %} + - user: socore + - name: '/usr/sbin/so-yara-update >> /opt/so/log/yarasync/yara-update.log 2>&1' + - identifier: strelka-yara-update + - hour: '7' + - minute: '1' + strelka-yara-download: + {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} + cron.present: + {% else %} + cron.absent: + {% endif %} + - user: socore + - name: '/usr/sbin/so-yara-download >> /opt/so/log/yarasync/yara-download.log 2>&1' + - identifier: strelka-yara-download + - hour: '7' + - minute: '1' + {% if not GLOBALS.airgap %} + update_yara_rules: + cmd.run: + - name: /usr/sbin/so-yara-update + - onchanges: + - file: yara_update_scripts + download_yara_rules: + cmd.run: + - name: /usr/sbin/so-yara-download + - onchanges: + - file: yara_update_scripts + {% endif %} + {% endif %} + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/strelka/backend/config.sls b/salt/strelka/backend/config.sls index b39e06ac8..db18a68cc 100644 --- a/salt/strelka/backend/config.sls +++ b/salt/strelka/backend/config.sls @@ -50,6 +50,16 @@ backend_taste: - user: 939 - group: 939 +{% if STRELKAMERGED.rules.enabled %} +strelkarules: + file.recurse: + - name: /opt/so/conf/strelka/rules + - source: salt://strelka/rules + - user: 939 + - group: 939 + - clean: True +{% endif %} + {% else %} {{sls}}_state_not_allowed: From fc66a549027070ab75eecd9fda9d7fbb7aee074a Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 8 Mar 2024 16:26:14 +0000 Subject: [PATCH 117/140] Add Strelka download and update scripts back --- .../manager/tools/sbin_jinja/so-yara-download | 51 +++++++++++++++++++ salt/manager/tools/sbin_jinja/so-yara-update | 41 +++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 salt/manager/tools/sbin_jinja/so-yara-download create mode 100644 salt/manager/tools/sbin_jinja/so-yara-update diff --git a/salt/manager/tools/sbin_jinja/so-yara-download b/salt/manager/tools/sbin_jinja/so-yara-download new file mode 100644 index 000000000..aa9576253 --- /dev/null +++ b/salt/manager/tools/sbin_jinja/so-yara-download @@ -0,0 +1,51 @@ +#!/bin/bash +NOROOT=1 +. /usr/sbin/so-common + +{%- set proxy = salt['pillar.get']('manager:proxy') %} +{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} + +# Download the rules from the internet +{%- if proxy %} +export http_proxy={{ proxy }} +export https_proxy={{ proxy }} +export no_proxy="{{ noproxy }}" +{%- endif %} + +repos="/opt/so/conf/strelka/repos.txt" +output_dir=/nsm/rules/yara +gh_status=$(curl -s -o /dev/null -w "%{http_code}" https://github.com) +clone_dir="/tmp" +if [ "$gh_status" == "200" ] || [ "$gh_status" == "301" ]; then + + while IFS= read -r repo; do + if ! $(echo "$repo" | grep -qE '^#'); then + # Remove old repo if existing bc of previous error condition or unexpected disruption + repo_name=`echo $repo | awk -F '/' '{print $NF}'` + [ -d $output_dir/$repo_name ] && rm -rf $output_dir/$repo_name + + # Clone repo and make appropriate directories for rules + git clone $repo $clone_dir/$repo_name + echo "Analyzing rules from $clone_dir/$repo_name..." + mkdir -p $output_dir/$repo_name + # Ensure a copy of the license is available for the rules + [ -f $clone_dir/$repo_name/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name + + # Copy over rules + for i in $(find $clone_dir/$repo_name -name "*.yar*"); do + rule_name=$(echo $i | awk -F '/' '{print $NF}') + cp $i $output_dir/$repo_name + done + rm -rf $clone_dir/$repo_name + fi + done < $repos + + echo "Done!" + +/usr/sbin/so-yara-update + +else + echo "Server returned $gh_status status code." + echo "No connectivity to Github...exiting..." + exit 1 +fi diff --git a/salt/manager/tools/sbin_jinja/so-yara-update b/salt/manager/tools/sbin_jinja/so-yara-update new file mode 100644 index 000000000..07c940f47 --- /dev/null +++ b/salt/manager/tools/sbin_jinja/so-yara-update @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +NOROOT=1 +. /usr/sbin/so-common + +echo "Starting to check for yara rule updates at $(date)..." + +newcounter=0 +excludedcounter=0 +excluded_rules=({{ EXCLUDEDRULES | join(' ') }}) + +# Pull down the SO Rules +SORULEDIR=/nsm/rules/yara +OUTPUTDIR=/opt/so/saltstack/local/salt/strelka/rules + +mkdir -p $OUTPUTDIR +# remove all rules prior to copy so we can clear out old rules +rm -f $OUTPUTDIR/* + +for i in $(find $SORULEDIR -name "*.yar" -o -name "*.yara"); do + rule_name=$(echo $i | awk -F '/' '{print $NF}') + if [[ ! "${excluded_rules[*]}" =~ ${rule_name} ]]; then + echo "Adding rule: $rule_name..." + cp $i $OUTPUTDIR/$rule_name + ((newcounter++)) + else + echo "Excluding rule: $rule_name..." + ((excludedcounter++)) + fi +done + +if [ "$newcounter" -gt 0 ] || [ "$excludedcounter" -gt 0 ];then + echo "$newcounter rules added." + echo "$excludedcounter rule(s) excluded." +fi + +echo "Finished rule updates at $(date)..." From e8ae60901233b06742a915cb64940e8564d75f37 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 8 Mar 2024 16:27:17 +0000 Subject: [PATCH 118/140] Add Strelka rules watch back --- salt/strelka/backend/enabled.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/strelka/backend/enabled.sls b/salt/strelka/backend/enabled.sls index 9ebb1a148..fc56f4197 100644 --- a/salt/strelka/backend/enabled.sls +++ b/salt/strelka/backend/enabled.sls @@ -42,8 +42,8 @@ strelka_backend: {% endfor %} {% endif %} - restart_policy: on-failure - #- watch: - # - file: strelkarules + - watch: + - file: strelkarules delete_so-strelka-backend_so-status.disabled: file.uncomment: From 6680e023e4f9dcec593d78967c9f6a49eba63856 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 8 Mar 2024 12:16:59 -0500 Subject: [PATCH 119/140] Update soc_pcap.yaml --- salt/pcap/soc_pcap.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/pcap/soc_pcap.yaml b/salt/pcap/soc_pcap.yaml index 32204a23a..96bc3831d 100644 --- a/salt/pcap/soc_pcap.yaml +++ b/salt/pcap/soc_pcap.yaml @@ -4,32 +4,32 @@ pcap: helpLink: stenographer.html config: maxdirectoryfiles: - description: The maximum number of packet/index files to create before deleting old files. + description: By default, Stenographer limits the number of files in the pcap directory to 30000 to avoid limitations with the ext3 filesystem. However, if you're using the ext4 or xfs filesystems, then it is safe to increase this value. So if you have a large amount of storage and find that you only have 3 weeks worth of PCAP on disk while still having plenty of free space, then you may want to increase this default setting. helpLink: stenographer.html diskfreepercentage: - description: The disk space percent to always keep free for PCAP + description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated forward nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and :ref:`elasticsearch` at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21. helpLink: stenographer.html blocks: - description: The number of 1MB packet blocks used by AF_PACKET to store packets in memory, per thread. You shouldn't need to change this. + description: The number of 1MB packet blocks used by Stenographer and AF_PACKET to store packets in memory, per thread. You shouldn't need to change this. advanced: True helpLink: stenographer.html preallocate_file_mb: - description: File size to pre-allocate for individual PCAP files. You shouldn't need to change this. + description: File size to pre-allocate for individual Stenographer PCAP files. You shouldn't need to change this. advanced: True helpLink: stenographer.html aiops: - description: The max number of async writes to allow at once. + description: The max number of async writes to allow for Stenographer at once. advanced: True helpLink: stenographer.html pin_to_cpu: - description: Enable CPU pinning for PCAP. + description: Enable CPU pinning for Stenographer PCAP. advanced: True helpLink: stenographer.html cpus_to_pin_to: - description: CPU to pin PCAP to. Currently only a single CPU is supported. + description: CPU to pin Stenographer PCAP to. Currently only a single CPU is supported. advanced: True helpLink: stenographer.html disks: - description: List of disks to use for PCAP. This is currently not used. + description: List of disks to use for Stenographer PCAP. This is currently not used. advanced: True helpLink: stenographer.html From 6f05c3976b45337fae4699c953345a80179b398b Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 8 Mar 2024 11:29:46 -0700 Subject: [PATCH 120/140] Updated RulesRepo for New Strelka Structure --- salt/soc/defaults.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 5699c7722..7be2db772 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1132,7 +1132,8 @@ soc: compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py reposFolder: /opt/sensoroni/yara/repos rulesRepos: - - https://github.com/Security-Onion-Solutions/securityonion-yara + - repo: https://github.com/Security-Onion-Solutions/securityonion-yara + license: DRL yaraRulesFolder: /opt/sensoroni/yara/rules suricataengine: communityRulesFile: /nsm/rules/suricata/emerging-all.rules From a55e04e64a0fe67ec20b2813df0aec7fa679ad1b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 8 Mar 2024 15:48:53 -0500 Subject: [PATCH 121/140] pcap improvements --- salt/pcap/config.sls | 7 ------- salt/pcap/init.sls | 9 +++++++++ salt/sensoroni/enabled.sls | 3 --- salt/sensoroni/files/sensoroni.json | 12 +----------- salt/suricata/enabled.sls | 5 ----- salt/suricata/init.sls | 1 + salt/suricata/pcap.sls | 10 ++++++++-- 7 files changed, 19 insertions(+), 28 deletions(-) diff --git a/salt/pcap/config.sls b/salt/pcap/config.sls index 9ea5cee65..eb37765c5 100644 --- a/salt/pcap/config.sls +++ b/salt/pcap/config.sls @@ -72,13 +72,6 @@ stenoca: - user: 941 - group: 939 -pcapdir: - file.directory: - - name: /nsm/pcap - - user: 941 - - group: 941 - - makedirs: True - pcaptmpdir: file.directory: - name: /nsm/pcaptmp diff --git a/salt/pcap/init.sls b/salt/pcap/init.sls index 9de272ad7..7a172e8fd 100644 --- a/salt/pcap/init.sls +++ b/salt/pcap/init.sls @@ -15,3 +15,12 @@ include: {% else %} - pcap.disabled {% endif %} + +# This directory needs to exist regardless of whether STENO is enabled or not, in order for +# Sensoroni to be able to look at old steno PCAP data +pcapdir: + file.directory: + - name: /nsm/pcap + - user: 941 + - group: 941 + - makedirs: True \ No newline at end of file diff --git a/salt/sensoroni/enabled.sls b/salt/sensoroni/enabled.sls index 6dc3df2bd..3f05568a0 100644 --- a/salt/sensoroni/enabled.sls +++ b/salt/sensoroni/enabled.sls @@ -23,10 +23,7 @@ so-sensoroni: - /opt/so/conf/sensoroni/sensoroni.json:/opt/sensoroni/sensoroni.json:ro - /opt/so/conf/sensoroni/analyzers:/opt/sensoroni/analyzers:rw - /opt/so/log/sensoroni:/opt/sensoroni/logs:rw - {% if GLOBALS.pcap_engine == "SURICATA" %} - /nsm/suripcap/:/nsm/suripcap:rw - - /nsm/suripcaptmp:/nsm/suripcaptmp:rw - {% endif %} {% if DOCKER.containers['so-sensoroni'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-sensoroni'].custom_bind_mounts %} - {{ BIND }} diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index eb9c1131d..f40f73167 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -23,29 +23,19 @@ "statickeyauth": { "apiKey": "{{ GLOBALS.sensoroni_key }}" {% if GLOBALS.is_sensor %} -{% from 'pcap/config.map.jinja' import PCAPMERGED %} -{% from 'suricata/map.jinja' import SURICATAMERGED %} -{# if PCAPMERGED.enabled is true then we know that steno is the pcap engine #} -{# if it is false, then user has steno disabled in ui or has selected suricata for pcap engine #} -{%- if PCAPMERGED.enabled %} }, "stenoquery": { "executablePath": "/opt/sensoroni/scripts/stenoquery.sh", "pcapInputPath": "/nsm/pcap", "pcapOutputPath": "/nsm/pcapout" - } -{%- elif GLOBALS.pcap_engine == "SURICATA" and SURICATAMERGED.enabled %} }, "suriquery": { "executablePath": "/opt/sensoroni/scripts/suriquery.sh", "pcapInputPath": "/nsm/suripcap", "pcapOutputPath": "/nsm/pcapout", "pcapMaxCount": {{ SENSORONIMERGED.config.suripcap.pcapMaxCount }} - } -{% endif %} -{%- else %} - } {%- endif %} + } } } } diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index fed5783e1..d35160527 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -12,9 +12,6 @@ include: - suricata.config - suricata.sostatus -{% if GLOBALS.pcap_engine == "SURICATA" %} - - suricata.pcap -{% endif %} so-suricata: docker_container.running: @@ -36,9 +33,7 @@ so-suricata: - /nsm/suricata/:/nsm/:rw - /nsm/suricata/extracted:/var/log/suricata//filestore:rw - /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro - {% if GLOBALS.pcap_engine == "SURICATA" %} - /nsm/suripcap/:/nsm/suripcap:rw - {% endif %} {% if DOCKER.containers['so-suricata'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-suricata'].custom_bind_mounts %} - {{ BIND }} diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index 64a000109..f6ddce862 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -7,6 +7,7 @@ {% from 'suricata/map.jinja' import SURICATAMERGED %} include: + - suricata.pcap {% if SURICATAMERGED.enabled and GLOBALS.role != 'so-import' %} - suricata.enabled {% elif GLOBALS.role == 'so-import' %} diff --git a/salt/suricata/pcap.sls b/salt/suricata/pcap.sls index a3cbafa0a..665262477 100644 --- a/salt/suricata/pcap.sls +++ b/salt/suricata/pcap.sls @@ -1,14 +1,18 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'suricata/map.jinja' import SURICATAMERGED %} +# This directory needs to exist regardless of whether SURIPCAP is enabled or not, in order for +# Sensoroni to be able to look at old Suricata PCAP data suripcapdir: file.directory: - name: /nsm/suripcap - user: 940 - group: 939 - - mode: 755 + - mode: 775 - makedirs: True +{% if GLOBALS.pcap_engine == "SURICATA" %} + {# there should only be 1 interface in af-packet so we can just reference the first list item #} {% for i in range(1, SURICATAMERGED.config['af-packet'][0].threads + 1) %} @@ -17,6 +21,8 @@ suripcapthread{{i}}dir: - name: /nsm/suripcap/{{i}} - user: 940 - group: 939 - - mode: 755 + - mode: 775 {% endfor %} + +{% endif %} From a892352b612627c05ae83da48eb0bbc383c27d9c Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 8 Mar 2024 16:43:29 -0500 Subject: [PATCH 122/140] Update soc_pcap.yaml --- salt/pcap/soc_pcap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/pcap/soc_pcap.yaml b/salt/pcap/soc_pcap.yaml index 96bc3831d..65fb99d86 100644 --- a/salt/pcap/soc_pcap.yaml +++ b/salt/pcap/soc_pcap.yaml @@ -7,7 +7,7 @@ pcap: description: By default, Stenographer limits the number of files in the pcap directory to 30000 to avoid limitations with the ext3 filesystem. However, if you're using the ext4 or xfs filesystems, then it is safe to increase this value. So if you have a large amount of storage and find that you only have 3 weeks worth of PCAP on disk while still having plenty of free space, then you may want to increase this default setting. helpLink: stenographer.html diskfreepercentage: - description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated forward nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and :ref:`elasticsearch` at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21. + description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated forward nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and Elasticsearch at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21. helpLink: stenographer.html blocks: description: The number of 1MB packet blocks used by Stenographer and AF_PACKET to store packets in memory, per thread. You shouldn't need to change this. From 34d5954e169972e21e412fb236f8bfc80cca788e Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 11 Mar 2024 09:12:05 -0400 Subject: [PATCH 123/140] Fix indent --- salt/manager/init.sls | 86 +++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index ee564dce8..c62a41999 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -118,49 +118,49 @@ rules_dir: - makedirs: True {% if STRELKAMERGED.rules.enabled %} - strelkarepos: - file.managed: - - name: /opt/so/conf/strelka/repos.txt - - source: salt://strelka/rules/repos.txt.jinja - - template: jinja - - defaults: - STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} - - makedirs: True - strelka-yara-update: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-update >> /opt/so/log/yarasync/yara-update.log 2>&1' - - identifier: strelka-yara-update - - hour: '7' - - minute: '1' - strelka-yara-download: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-download >> /opt/so/log/yarasync/yara-download.log 2>&1' - - identifier: strelka-yara-download - - hour: '7' - - minute: '1' - {% if not GLOBALS.airgap %} - update_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-update - - onchanges: - - file: yara_update_scripts - download_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-download - - onchanges: - - file: yara_update_scripts - {% endif %} - {% endif %} +strelkarepos: + file.managed: + - name: /opt/so/conf/strelka/repos.txt + - source: salt://strelka/rules/repos.txt.jinja + - template: jinja + - defaults: + STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} + - makedirs: True +strelka-yara-update: + {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} + cron.present: + {% else %} + cron.absent: + {% endif %} + - user: socore + - name: '/usr/sbin/so-yara-update >> /opt/so/log/yarasync/yara-update.log 2>&1' + - identifier: strelka-yara-update + - hour: '7' + - minute: '1' +strelka-yara-download: + {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} + cron.present: + {% else %} + cron.absent: + {% endif %} + - user: socore + - name: '/usr/sbin/so-yara-download >> /opt/so/log/yarasync/yara-download.log 2>&1' + - identifier: strelka-yara-download + - hour: '7' + - minute: '1' +{% if not GLOBALS.airgap %} +update_yara_rules: + cmd.run: + - name: /usr/sbin/so-yara-update + - onchanges: + - file: yara_update_scripts +download_yara_rules: + cmd.run: + - name: /usr/sbin/so-yara-download + - onchanges: + - file: yara_update_scripts +{% endif %} +{% endif %} {% else %} From a8403c63c73d37ad97ee0e5565fe8c3109c4019c Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 11 Mar 2024 09:35:54 -0400 Subject: [PATCH 124/140] Create local salt dir for stig Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/manager/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 8b5d19751..028931012 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -537,6 +537,8 @@ up_to_2.4.40() { up_to_2.4.50() { echo "Creating additional pillars.." mkdir -p /opt/so/saltstack/local/pillar/stig/ + mkdir -p /opt/so/saltstack/local/salt/stig/ + chown socore:socore /opt/so/saltstack/local/salt/stig/ touch /opt/so/saltstack/local/pillar/stig/adv_stig.sls touch /opt/so/saltstack/local/pillar/stig/soc_stig.sls From 907cf9f9924d25c3e309a32900c9a522ffa8a212 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Mar 2024 12:20:28 -0400 Subject: [PATCH 125/140] transition pcap --- salt/bpf/pcap.map.jinja | 17 ++++++++++------- salt/global/soc_global.yaml | 6 +++--- salt/suricata/map.jinja | 2 +- salt/suricata/pcap.sls | 2 +- salt/telegraf/scripts/oldpcap.sh | 2 +- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/salt/bpf/pcap.map.jinja b/salt/bpf/pcap.map.jinja index c1d7562cc..a6deae4f4 100644 --- a/salt/bpf/pcap.map.jinja +++ b/salt/bpf/pcap.map.jinja @@ -1,7 +1,10 @@ -{% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %} -{% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %} -{% import 'bpf/macros.jinja' as MACROS %} - -{{ MACROS.remove_comments(BPFMERGED, 'pcap') }} - -{% set PCAPBPF = BPFMERGED.pcap %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% if GLOBALS.pcap_engine == "TRANSITION" %} +{% set PCAPBPF = "ip and host 255.255.255.1 and port 1" %} +{% else %} +{% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %} +{% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %} +{% import 'bpf/macros.jinja' as MACROS %} +{{ MACROS.remove_comments(BPFMERGED, 'pcap') }} +{% set PCAPBPF = BPFMERGED.pcap %} +{% endif %} diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index d707fb1cc..a48476214 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -15,9 +15,9 @@ global: regexFailureMessage: You must enter either ZEEK or SURICATA. global: True pcapengine: - description: Which engine to use for generating pcap. Options are STENO and SURICATA. - regex: ^(STENO|SURICATA)$ - regexFailureMessage: You must enter either STENO or SURICATA. + description: Which engine to use for generating pcap. Options are STENO, SURICATA or TRANSITION. + regex: ^(STENO|SURICATA|TRANSITION)$ + regexFailureMessage: You must enter either STENO, SURICATA or TRANSITION. global: True ids: description: Which IDS engine to use. Currently only Suricata is supported. diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 6ba3c3b73..7f7b04aef 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -9,7 +9,7 @@ {% set surimeta_filestore_index = [] %} {# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #} -{% if GLOBALS.pcap_engine == "SURICATA" %} +{% if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %} {# move the items in suricata.pcap into suricata.config.outputs.pcap-log. these items were placed under suricata.config for ease of access in SOC #} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'compression': SURICATAMERGED.pcap.compression}) %} diff --git a/salt/suricata/pcap.sls b/salt/suricata/pcap.sls index 665262477..87b568f96 100644 --- a/salt/suricata/pcap.sls +++ b/salt/suricata/pcap.sls @@ -11,7 +11,7 @@ suripcapdir: - mode: 775 - makedirs: True -{% if GLOBALS.pcap_engine == "SURICATA" %} +{% if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %} {# there should only be 1 interface in af-packet so we can just reference the first list item #} {% for i in range(1, SURICATAMERGED.config['af-packet'][0].threads + 1) %} diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index 438ce912c..876ff7835 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -5,7 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- if GLOBALS.pcap_engine == "SURICATA" %} +{%- if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %} PCAPLOC=/host/nsm/suripcap {%- else %} PCAPLOC=/host/nsm/pcap From b5d8df7fb2ab72420d4fd35efe71cd9f3ead586b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 11 Mar 2024 13:45:57 -0400 Subject: [PATCH 126/140] auto-convert email addresses to lowercase during setup --- setup/so-whiptail | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index ede138d26..5e2a2de0e 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -195,10 +195,12 @@ whiptail_create_web_user() { [ -n "$TESTING" ] && return WEBUSER=$(whiptail --title "$whiptail_title" --inputbox \ - "Please enter an email address to create an administrator account for the Security Onion Console (SOC) web interface.\n\nThis will also be used for Elasticsearch and Kibana." 12 60 "$1" 3>&1 1>&2 2>&3) + "Please enter an email address to create an administrator account for the Security Onion Console (SOC) web interface.\n\nThis will also be used for Elasticsearch and Kibana.\n\nMust only include letters, numbers, or + - _ % . @ characters. All capitalized letters will be converted to lowercase." 12 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus + + WEBUSER=${WEBUSER,,} } whiptail_create_web_user_password1() { From cd28c00d67dd658bf247e8d3c1401f35c67f7e80 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 11 Mar 2024 13:47:31 -0400 Subject: [PATCH 127/140] auto-convert email addresses to lowercase during setup --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 5e2a2de0e..ff8c9fe8d 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -195,7 +195,7 @@ whiptail_create_web_user() { [ -n "$TESTING" ] && return WEBUSER=$(whiptail --title "$whiptail_title" --inputbox \ - "Please enter an email address to create an administrator account for the Security Onion Console (SOC) web interface.\n\nThis will also be used for Elasticsearch and Kibana.\n\nMust only include letters, numbers, or + - _ % . @ characters. All capitalized letters will be converted to lowercase." 12 60 "$1" 3>&1 1>&2 2>&3) + "Please enter an email address to create an administrator account for the Security Onion Console (SOC) web interface.\n\nThis will also be used for Elasticsearch and Kibana.\n\nMust only include letters, numbers, or + - _ % . @ characters. All capitalized letters will be converted to lowercase." 15 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From ba32b3e6e9d23a7c34fadef272f5bf8ec2e52ae3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Mar 2024 14:07:45 -0400 Subject: [PATCH 128/140] fix bpf for transition --- salt/bpf/pcap.map.jinja | 2 +- salt/soc/defaults.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/bpf/pcap.map.jinja b/salt/bpf/pcap.map.jinja index a6deae4f4..4d8fef460 100644 --- a/salt/bpf/pcap.map.jinja +++ b/salt/bpf/pcap.map.jinja @@ -1,6 +1,6 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% if GLOBALS.pcap_engine == "TRANSITION" %} -{% set PCAPBPF = "ip and host 255.255.255.1 and port 1" %} +{% set PCAPBPF = ["ip and host 255.255.255.1 and port 1"] %} {% else %} {% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %} {% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %} diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 5699c7722..7be2db772 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1132,7 +1132,8 @@ soc: compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py reposFolder: /opt/sensoroni/yara/repos rulesRepos: - - https://github.com/Security-Onion-Solutions/securityonion-yara + - repo: https://github.com/Security-Onion-Solutions/securityonion-yara + license: DRL yaraRulesFolder: /opt/sensoroni/yara/rules suricataengine: communityRulesFile: /nsm/rules/suricata/emerging-all.rules From 61a183b7fc567ebdce5f9252a6af87d0330fda19 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Mon, 11 Mar 2024 15:55:39 -0400 Subject: [PATCH 129/140] Add regex defaults --- salt/soc/defaults.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7be2db772..197aee070 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1078,8 +1078,10 @@ soc: kratos: hostUrl: elastalertengine: + allowRegex: '' autoUpdateEnabled: false - communityRulesImportFrequencySeconds: 180 + communityRulesImportFrequencySeconds: 86400 + denyRegex: '.*' elastAlertRulesFolder: /opt/sensoroni/elastalert rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint sigmaRulePackages: @@ -1128,15 +1130,19 @@ soc: userFiles: - rbac/users_roles strelkaengine: + allowRegex: '' autoUpdateEnabled: false compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py + denyRegex: '.*' reposFolder: /opt/sensoroni/yara/repos rulesRepos: - repo: https://github.com/Security-Onion-Solutions/securityonion-yara license: DRL yaraRulesFolder: /opt/sensoroni/yara/rules suricataengine: + allowRegex: '' communityRulesFile: /nsm/rules/suricata/emerging-all.rules + denyRegex: '.*' rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint client: enableReverseLookup: false From 72acb11925bd85afcbd1fc59f167ff96520fb253 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 11 Mar 2024 19:04:51 -0400 Subject: [PATCH 130/140] Update soc_suricata.yaml --- salt/suricata/soc_suricata.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index da7586e97..c61c04123 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -59,8 +59,8 @@ suricata: regexFailureMessage: You must enter either yes or no. helpLink: suricata.html conditional: - description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules. - regex: ^(all|alert|tag)$ + description: Set to "all" to capture PCAP for all flows. Set to "alerts" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules. + regex: ^(all|alerts|tag)$ regexFailureMessage: You must enter either all, alert or tag. helpLink: suricata.html dir: From 3e0fb3f8bb953f32dde333b25ad35266689e6fbc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 12 Mar 2024 10:18:27 -0400 Subject: [PATCH 131/140] Update so-saltstack-update --- salt/manager/tools/sbin/so-saltstack-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-saltstack-update b/salt/manager/tools/sbin/so-saltstack-update index b15fce008..4be8f095c 100755 --- a/salt/manager/tools/sbin/so-saltstack-update +++ b/salt/manager/tools/sbin/so-saltstack-update @@ -47,7 +47,7 @@ got_root(){ got_root if [ $# -ne 1 ] ; then - BRANCH=master + BRANCH=2.4/main else BRANCH=$1 fi From 06013e2c6fd75d04cab6fda7ccc10c455e2c36f5 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 13 Mar 2024 07:23:43 -0400 Subject: [PATCH 132/140] Gen packages post-SOUP --- salt/manager/tools/sbin/soup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 028931012..a585f877c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -430,7 +430,8 @@ post_to_2.4.50() { } post_to_2.4.60() { - echo "Nothing to apply" + echo "Regenerating Elastic Agent Installers..." + so-elastic-agent-gen-installers POSTVERSION=2.4.60 } From 1a829190ac4ba8777ac68bc865bfc6944be55d92 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 13 Mar 2024 09:46:44 -0400 Subject: [PATCH 133/140] remove modules if detections disabled --- salt/soc/defaults.yaml | 2 +- salt/soc/merged.map.jinja | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 197aee070..de372a98f 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1081,7 +1081,7 @@ soc: allowRegex: '' autoUpdateEnabled: false communityRulesImportFrequencySeconds: 86400 - denyRegex: '.*' + denyRegex: '' elastAlertRulesFolder: /opt/sensoroni/elastalert rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint sigmaRulePackages: diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index bc7c5cada..2012917af 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -30,6 +30,13 @@ {# since cases is not a valid soc config item and only used for the map files, remove it from being placed in the config #} {% do SOCMERGED.config.server.modules.pop('cases') %} +{# remove these modules if detections is disabled #} +{% if not SOCMERGED.config.server.client.detectionsEnabled %} +{% do SOCMERGED.config.server.modules.pop('elastalertengine') %} +{% do SOCMERGED.config.server.modules.pop('strelkaengine') %} +{% do SOCMERGED.config.server.modules.pop('suricataengine') %} +{% endif %} + {% if pillar.manager.playbook == 0 %} {% do SOCMERGED.config.server.client.inactiveTools.append('toolPlaybook') %} {% endif %} From b9702d02db13c83765894067c9b07732ee41eff7 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 13 Mar 2024 11:24:26 -0400 Subject: [PATCH 134/140] Update init.sls --- salt/salt/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/salt/init.sls b/salt/salt/init.sls index a190a84eb..b2ea31a65 100644 --- a/salt/salt/init.sls +++ b/salt/salt/init.sls @@ -10,3 +10,4 @@ salt_bootstrap: - name: /usr/sbin/bootstrap-salt.sh - source: salt://salt/scripts/bootstrap-salt.sh - mode: 755 + - show_changes: False From 275a678fa1a90369d2d64ff2260679975384f548 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 13 Mar 2024 13:49:44 -0400 Subject: [PATCH 135/140] removed unused property --- salt/sensoroni/files/sensoroni.json | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index f40f73167..547e52ada 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -30,7 +30,6 @@ "pcapOutputPath": "/nsm/pcapout" }, "suriquery": { - "executablePath": "/opt/sensoroni/scripts/suriquery.sh", "pcapInputPath": "/nsm/suripcap", "pcapOutputPath": "/nsm/pcapout", "pcapMaxCount": {{ SENSORONIMERGED.config.suripcap.pcapMaxCount }} From 927fe9039d1dca052e96cbfcdd3db380fe49b672 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 13 Mar 2024 20:50:03 -0400 Subject: [PATCH 136/140] handle airgap when detections not enabled --- salt/soc/merged.map.jinja | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index 2012917af..57abe7a48 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -35,18 +35,18 @@ {% do SOCMERGED.config.server.modules.pop('elastalertengine') %} {% do SOCMERGED.config.server.modules.pop('strelkaengine') %} {% do SOCMERGED.config.server.modules.pop('suricataengine') %} +{% elif pillar.global.airgap %} + {# if system is Airgap, don't autoupdate Yara & Sigma rules #} + {% do SOCMERGED.config.server.modules.elastalertengine.update({'autoUpdateEnabled': false}) %} + {% do SOCMERGED.config.server.modules.strelkaengine.update({'autoUpdateEnabled': false}) %} +{% endif %} + {% endif %} {% if pillar.manager.playbook == 0 %} {% do SOCMERGED.config.server.client.inactiveTools.append('toolPlaybook') %} {% endif %} -{# if system is Airgap, don't autoupdate Yara & Sigma rules #} -{% if pillar.global.airgap %} - {% do SOCMERGED.config.server.modules.elastalertengine.update({'autoUpdateEnabled': false}) %} - {% do SOCMERGED.config.server.modules.strelkaengine.update({'autoUpdateEnabled': false}) %} -{% endif %} - {% set standard_actions = SOCMERGED.config.pop('actions') %} {% if pillar.global.endgamehost != '' %} From 844cfe55cd0ea40317a51f9cee33a801d690d647 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 13 Mar 2024 20:52:17 -0400 Subject: [PATCH 137/140] handle airgap when detections not enabled --- salt/soc/merged.map.jinja | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index 57abe7a48..c22ed2210 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -41,8 +41,6 @@ {% do SOCMERGED.config.server.modules.strelkaengine.update({'autoUpdateEnabled': false}) %} {% endif %} -{% endif %} - {% if pillar.manager.playbook == 0 %} {% do SOCMERGED.config.server.client.inactiveTools.append('toolPlaybook') %} {% endif %} From 284e0d84354a0d8b769a4582d0c68e52b749e669 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 14 Mar 2024 11:33:47 -0400 Subject: [PATCH 138/140] Update soc_suricata.yaml --- salt/suricata/soc_suricata.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index c61c04123..47e9e1503 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -28,19 +28,16 @@ suricata: description: Size in GB for total usage size of PCAP on disk. helplink: suricata.html compression: - description: Enable compression of Suricata PCAP. Currently unsupported + description: Enable compression of Suricata PCAP. advanced: True - readonly: True helpLink: suricata.html lz4-checksum: - description: Enable PCAP lz4 checksum. Currently unsupported + description: Enable PCAP lz4 checksum. advanced: True - readonly: True helpLink: suricata.html lz4-level: description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported advanced: True - readonly: True helpLink: suricata.html filename: description: Filename output for Suricata PCAP. From fd835f63947b152d3390c3b6f866c9f0e7c338a2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 14 Mar 2024 11:36:45 -0400 Subject: [PATCH 139/140] Update soc_suricata.yaml --- salt/suricata/soc_suricata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 47e9e1503..7decaa6d3 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -36,7 +36,7 @@ suricata: advanced: True helpLink: suricata.html lz4-level: - description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported + description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. advanced: True helpLink: suricata.html filename: From bb3bbd749c65ee45489b847a587e6be25a419d8a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 20 Mar 2024 10:20:04 -0400 Subject: [PATCH 140/140] 2.4.260 --- DOWNLOAD_AND_VERIFY_ISO.md | 22 ++++++++++----------- sigs/securityonion-2.4.60-20240320.iso.sig | Bin 0 -> 566 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-2.4.60-20240320.iso.sig diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index a23d88d4d..4493f210d 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,17 +1,17 @@ -### 2.4.50-20240220 ISO image released on 2024/02/20 +### 2.4.60-20240320 ISO image released on 2024/03/20 ### Download and Verify -2.4.50-20240220 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.4.50-20240220.iso +2.4.60-20240320 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.4.60-20240320.iso -MD5: BCA6476EF1BF79773D8EFB11700FDE8E -SHA1: 9FF0A304AA368BCD2EF2BE89AD47E65650241927 -SHA256: 49D7695EFFF6F3C4840079BF564F3191B585639816ADE98672A38017F25E9570 +MD5: 178DD42D06B2F32F3870E0C27219821E +SHA1: 73EDCD50817A7F6003FE405CF1808A30D034F89D +SHA256: DD334B8D7088A7B78160C253B680D645E25984BA5CCAB5CC5C327CA72137FC06 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.50-20240220.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.60-20240320.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS @@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2. Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.50-20240220.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.60-20240320.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.4.50-20240220.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.4.60-20240320.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.4.50-20240220.iso.sig securityonion-2.4.50-20240220.iso +gpg --verify securityonion-2.4.60-20240320.iso.sig securityonion-2.4.60-20240320.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Fri 16 Feb 2024 11:36:25 AM EST using RSA key ID FE507013 +gpg: Signature made Tue 19 Mar 2024 03:17:58 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.4.60-20240320.iso.sig b/sigs/securityonion-2.4.60-20240320.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..c0129ab64b049b783258081c8d3dde117b29aa95 GIT binary patch literal 566 zcmV-60?GY}0y6{v0SEvc79j-41gSkXz6^6dp_W8^5Ma0dP;e6k0%iH+<^T!_5PT3| zxBgIY6FsdE0HMAsL4Ya2;dUH@Tj-=rYUOi%4z*NE^@q+`;Erl=A8#u|b^A0~xO#6`Szzg5~-3VjT3Yys$sGkuxpcsZ@^e5$v7pXryod-7`cIChZTfeuV& zKIlAHEhH+#f`k%z&qebjUlS~w3XH&a0N!TFO3A(4TzcH@Bj#jfvyTbV)Eyc8J;5eF zL9BEEpso{FY9)Vhc3*2LAuZ&Xo~l7+Jb{y4Qr$U&^WQTRe+g57T7=d8`o!9)8dp2s zgkP&e^vc+xnu!=S9`UpxO0R#HROR;d$ZLxxt`ZVR0Ne_ywWWs$w3}i184t&>AnQ#OU-=HH6mAF^^|dFa z>sH%7KeQ%8Gt+!Vx#jt6YUO&Y^g8?yMOnxK9R2w4q_eENgMup&e%TRy7{8${`LFyP$=x?8XlQVruAYhoUiD`V~ehpF literal 0 HcmV?d00001