From 5278601e5d1cd05a613a2486e867c95d87936d8c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 7 Aug 2023 11:18:35 -0400 Subject: [PATCH 01/56] manage telegraf scripts with a defaults file assigned per node type --- salt/telegraf/config.sls | 15 ++--- salt/telegraf/defaults.yaml | 79 ++++++++++++++++++++++ salt/telegraf/enabled.sls | 5 +- salt/telegraf/etc/telegraf.conf | 114 +++----------------------------- salt/telegraf/map.jinja | 10 ++- salt/telegraf/soc_telegraf.yaml | 19 +++++- 6 files changed, 127 insertions(+), 115 deletions(-) diff --git a/salt/telegraf/config.sls b/salt/telegraf/config.sls index 1cc7ceed0..0711260b5 100644 --- a/salt/telegraf/config.sls +++ b/salt/telegraf/config.sls @@ -32,17 +32,16 @@ tgrafetsdir: - name: /opt/so/conf/telegraf/scripts - makedirs: True -tgrafsyncscripts: - file.recurse: - - name: /opt/so/conf/telegraf/scripts +{% for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} +tgraf_sync_script_{{script}}: + file.managed: + - name: /opt/so/conf/telegraf/scripts/{{script}} - user: root - group: 939 - - file_mode: 770 + - mode: 770 - template: jinja - - source: salt://telegraf/scripts - {% if GLOBALS.md_engine == 'SURICATA' %} - - exclude_pat: zeekcaptureloss.sh - {% endif %} + - source: salt://telegraf/scripts/{{script}} +{% endfor %} telegraf_sbin: file.recurse: diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index 63d437763..36ef679f0 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -9,3 +9,82 @@ telegraf: flush_jitter: '0s' debug: 'false' quiet: 'false' + scripts: + eval: + - beatseps.sh + - checkfiles.sh + - influxdbsize.sh + - oldpcap.sh + - raid.sh + - redis.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + standalone: + - beatseps.sh + - checkfiles.sh + - eps.sh + - influxdbsize.sh + - oldpcap.sh + - raid.sh + - redis.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + manager: + - beatseps.sh + - influxdbsize.sh + - raid.sh + - redis.sh + - sostatus.sh + managersearch: + - beatseps.sh + - eps.sh + - influxdbsize.sh + - raid.sh + - redis.sh + - sostatus.sh + import: + - sostatus.sh + sensor: + - beatseps.sh + - checkfiles.sh + - oldpcap.sh + - raid.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + heavynode: + - beatseps.sh + - checkfiles.sh + - eps.sh + - oldpcap.sh + - raid.sh + - redis.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + idh: + - sostatus.sh + searchnode: + - beatseps.sh + - eps.sh + - raid.sh + - sostatus.sh + receiver: + - beatseps.sh + - eps.sh + - raid.sh + - redis.sh + - sostatus.sh + fleet: + - sostatus.sh + desktop: [] diff --git a/salt/telegraf/enabled.sls b/salt/telegraf/enabled.sls index 209c85fb0..598587e17 100644 --- a/salt/telegraf/enabled.sls +++ b/salt/telegraf/enabled.sls @@ -7,6 +7,7 @@ {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} +{% from 'telegraf/map.jinja' import TELEGRAFMERGED %} include: @@ -67,8 +68,10 @@ so-telegraf: {% endif %} - watch: - file: tgrafconf - - file: tgrafsyncscripts - file: node_config + {% for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} + - file: tgraf_sync_script_{{script}} + {% endfor %} - require: - file: tgrafconf - file: node_config diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 1a6cdc311..45b1283e0 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -193,7 +193,7 @@ username = "{{ ES_USER }}" password = "{{ ES_PASS }}" insecure_skip_verify = true -{%- elif grains['role'] in ['so-searchnode', 'so-hotnode', 'so-warmnode'] %} +{%- elif grains['role'] in ['so-searchnode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] cluster_stats = false @@ -244,6 +244,8 @@ {%- endif %} # # Read metrics from one or more commands that can output to stdout +{%- if 'sostatus.sh' in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} +{%- do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('sostatus.sh') %} [[inputs.exec]] commands = [ "/scripts/sostatus.sh" @@ -251,122 +253,26 @@ data_format = "influx" timeout = "15s" interval = "60s" +{%- endif %} -# ## Commands array -{% if grains['role'] in ['so-manager'] %} +{%- if TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] | length > 0 %} [[inputs.exec]] commands = [ - "/scripts/redis.sh", - "/scripts/influxdbsize.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" +{%- for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} + "/scripts/{{script}}"{% if not loop.last %},{% endif %} +{%- endfor %} ] data_format = "influx" ## Timeout for each command to complete. timeout = "15s" -{% elif grains['role'] in ['so-managersearch'] %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/influxdbsize.sh", - "/scripts/eps.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - ## Timeout for each command to complete. - timeout = "15s" -{% elif grains['role'] in ['so-searchnode', 'so-receiver'] %} -[[inputs.exec]] - commands = [ - "/scripts/eps.sh", - "/scripts/raid.sh", - {% if grains.role == 'so-receiver' %} - "/scripts/redis.sh", - {% endif %} - "/scripts/beatseps.sh" - ] - data_format = "influx" - ## Timeout for each command to complete. - timeout = "15s" -{% elif grains['role'] == 'so-sensor' %} -[[inputs.exec]] - commands = [ - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% elif grains['role'] == 'so-heavynode' %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/eps.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% elif grains['role'] == 'so-standalone' %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/influxdbsize.sh", - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/eps.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% elif grains['role'] == 'so-eval' %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/influxdbsize.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% endif %} +{%- endif %} {%- if salt['pillar.get']('healthcheck:enabled', False) %} [[inputs.file]] files = ["/host/nsm/zeek/logs/zeek_restart.log"] data_format = "influx" {%- endif %} + [[inputs.file]] files = ["/etc/telegraf/node_config.json"] name_override = "node_config" diff --git a/salt/telegraf/map.jinja b/salt/telegraf/map.jinja index f1412d3ac..0f197a7b1 100644 --- a/salt/telegraf/map.jinja +++ b/salt/telegraf/map.jinja @@ -2,6 +2,14 @@ or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at https://securityonion.net/license; you may not use this file except in compliance with the Elastic License 2.0. #} - + +{% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'telegraf/defaults.yaml' as TELEGRAFDEFAULTS %} {% set TELEGRAFMERGED = salt['pillar.get']('telegraf', TELEGRAFDEFAULTS.telegraf, merge=True) %} +{% from 'zeek/config.map.jinja' import ZEEKMERGED %} + +{# if the md engine isn't zeek or zeek is disabled, dont run the zeek scripts for telegraf #} +{% if (GLOBALS.role in ['so-eval', 'so-standalone', 'so-sensor', 'so-heavynode']) and (GLOBALS.md_engine != 'ZEEK' or not ZEEKMERGED.enabled) %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %} +{% endif %} diff --git a/salt/telegraf/soc_telegraf.yaml b/salt/telegraf/soc_telegraf.yaml index a688ea2a3..1550c66cb 100644 --- a/salt/telegraf/soc_telegraf.yaml +++ b/salt/telegraf/soc_telegraf.yaml @@ -42,4 +42,21 @@ telegraf: global: True advanced: True helpLink: telegraf.html - \ No newline at end of file + scripts: + eval: &telegrafscripts + description: List of input.exec scripts to run for this node type. The script must be present in salt/telegraf/scripts. + forcedType: "[]string" + multiline: True + advanced: True + helpLink: telegraf.html + standalone: *telegrafscripts + manager: *telegrafscripts + managersearch: *telegrafscripts + import: *telegrafscripts + sensor: *telegrafscripts + heavynode: *telegrafscripts + idh: *telegrafscripts + searchnode: *telegrafscripts + receiver: *telegrafscripts + fleet: *telegrafscripts + desktop: *telegrafscripts From 8611d1848c8b09ab6b263288da0e6717bbaf9fda Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 7 Aug 2023 15:55:53 -0400 Subject: [PATCH 02/56] Set as default --- .../tools/sbin_jinja/so-elastic-fleet-es-url-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update index 6acda746c..975a8aec1 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update @@ -13,7 +13,7 @@ fi function update_es_urls() { # Generate updated JSON payload - JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"is_default":false,"is_default_monitoring":false,"config_yaml":""}') + JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}') # Update Fleet Elasticsearch URLs curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" From b9d0d032238cd926961348087ab773632bf754a4 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 7 Aug 2023 16:35:05 -0400 Subject: [PATCH 03/56] update version --- README.md | 4 ++-- VERSION | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index aa3aa6ddf..19a560419 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -## Security Onion 2.4 Release Candidate 2 (RC2) +## Security Onion 2.4 -Security Onion 2.4 Release Candidate 2 (RC2) is here! +Security Onion 2.4 is here! ## Screenshots diff --git a/VERSION b/VERSION index 59aa62c1f..b0f6bf0cd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.5 +2.4.10 From 609a2bf32e7085b18c0d039b0de51501800e468c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 09:27:03 -0400 Subject: [PATCH 04/56] only import ZEEKMERGED if a sensor type node --- salt/telegraf/map.jinja | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/salt/telegraf/map.jinja b/salt/telegraf/map.jinja index 0f197a7b1..e6d3460d6 100644 --- a/salt/telegraf/map.jinja +++ b/salt/telegraf/map.jinja @@ -6,10 +6,12 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'telegraf/defaults.yaml' as TELEGRAFDEFAULTS %} {% set TELEGRAFMERGED = salt['pillar.get']('telegraf', TELEGRAFDEFAULTS.telegraf, merge=True) %} -{% from 'zeek/config.map.jinja' import ZEEKMERGED %} -{# if the md engine isn't zeek or zeek is disabled, dont run the zeek scripts for telegraf #} -{% if (GLOBALS.role in ['so-eval', 'so-standalone', 'so-sensor', 'so-heavynode']) and (GLOBALS.md_engine != 'ZEEK' or not ZEEKMERGED.enabled) %} -{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %} -{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %} +{% if GLOBALS.role in ['so-eval', 'so-standalone', 'so-sensor', 'so-heavynode'] %} +{% from 'zeek/config.map.jinja' import ZEEKMERGED %} +{# if the md engine isn't zeek or zeek is disabled, dont run the zeek scripts for telegraf #} +{% if GLOBALS.md_engine != 'ZEEK' or not ZEEKMERGED.enabled %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %} +{% endif %} {% endif %} From 69553f9017b3ffe5ffa4ae776bf5006eddfec894 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 09:34:59 -0400 Subject: [PATCH 05/56] removes spaces from zeekcaptureloss script --- salt/telegraf/scripts/zeekcaptureloss.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index e254ada32..f2c3fcd2d 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -5,9 +5,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - - - # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp # if this script isn't already running From a06040c035e6db035426fb001afc1887276cc55a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 09:37:37 -0400 Subject: [PATCH 06/56] add WORKERS calculation back to zeekcaptureloss script --- salt/telegraf/scripts/zeekcaptureloss.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index 6e1785237..ddb6cd128 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -11,7 +11,11 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then if [ -d "/host/nsm/zeek/spool/logger" ]; then - WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }} +{%- if ZEEKMERGED.config.node.pins %} + WORKERS={{ ZEEKMERGED.config.node.pins | length }} +{%- else %} + WORKERS={{ ZEEKMERGED.config.node.lb_procs }} +{%- endif %} ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then WORKERS=1 From 673b45af092b6a655bf115edffc4d88ef0ba393a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 09:41:42 -0400 Subject: [PATCH 07/56] import ZEEKMERGED --- salt/telegraf/scripts/zeekcaptureloss.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index ddb6cd128..f2c3fcd2d 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -8,6 +8,7 @@ # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp # if this script isn't already running +{%- from 'zeek/config.map.jinja' import ZEEKMERGED %} if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then if [ -d "/host/nsm/zeek/spool/logger" ]; then From b2e75e77e8a079ee7066f0ab867691ea9cfb496f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 13:50:19 -0400 Subject: [PATCH 08/56] add local.rules and filter.rules to suricata defaults. add extraction.rules, local.rules and filter.rules for suricata metadata --- salt/suricata/defaults.yaml | 3 ++- salt/suricata/suricata_mdengine.yaml | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index 4651b7268..fd1b00929 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -416,7 +416,6 @@ suricata: enabled: "yes" filename: keyword_perf.log append: "yes" - prefilter: enabled: "yes" filename: prefilter_perf.log @@ -443,6 +442,8 @@ suricata: default-rule-path: /etc/suricata/rules rule-files: - all.rules + - local.rules + - filter.rules classification-file: /etc/suricata/classification.config reference-config-file: /etc/suricata/reference.config threshold-file: /etc/suricata/threshold.conf diff --git a/salt/suricata/suricata_mdengine.yaml b/salt/suricata/suricata_mdengine.yaml index 1c3855501..c6844541f 100644 --- a/salt/suricata/suricata_mdengine.yaml +++ b/salt/suricata/suricata_mdengine.yaml @@ -70,3 +70,9 @@ suricata: - flow #- netflow #- metadata + profiling: + rule-files: + - all.rules + - extraction.rules + - local.rules + - filter.rules From aab89d2483822359cc235827f3f4486024d3b288 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 13:54:58 -0400 Subject: [PATCH 09/56] rule-files does not go under profiling --- salt/suricata/suricata_mdengine.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/salt/suricata/suricata_mdengine.yaml b/salt/suricata/suricata_mdengine.yaml index c6844541f..80299dc5b 100644 --- a/salt/suricata/suricata_mdengine.yaml +++ b/salt/suricata/suricata_mdengine.yaml @@ -70,9 +70,8 @@ suricata: - flow #- netflow #- metadata - profiling: - rule-files: - - all.rules - - extraction.rules - - local.rules - - filter.rules + rule-files: + - all.rules + - extraction.rules + - local.rules + - filter.rules From 9118ac2b569e9fd3e3c994b24a8f6a4502d4331c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 13:59:43 -0400 Subject: [PATCH 10/56] filter.rules to filters.rules --- salt/suricata/defaults.yaml | 2 +- salt/suricata/suricata_mdengine.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index fd1b00929..4253794a8 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -443,7 +443,7 @@ suricata: rule-files: - all.rules - local.rules - - filter.rules + - filters.rules classification-file: /etc/suricata/classification.config reference-config-file: /etc/suricata/reference.config threshold-file: /etc/suricata/threshold.conf diff --git a/salt/suricata/suricata_mdengine.yaml b/salt/suricata/suricata_mdengine.yaml index 80299dc5b..d1fb7c2c3 100644 --- a/salt/suricata/suricata_mdengine.yaml +++ b/salt/suricata/suricata_mdengine.yaml @@ -74,4 +74,4 @@ suricata: - all.rules - extraction.rules - local.rules - - filter.rules + - filters.rules From 20dedab4b283cf3d7345170abd7ee24dac9bfbc3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 15:03:06 -0400 Subject: [PATCH 11/56] remove previously add rules files --- salt/suricata/defaults.yaml | 2 -- salt/suricata/suricata_mdengine.yaml | 5 ----- 2 files changed, 7 deletions(-) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index 4253794a8..050efa8f8 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -442,8 +442,6 @@ suricata: default-rule-path: /etc/suricata/rules rule-files: - all.rules - - local.rules - - filters.rules classification-file: /etc/suricata/classification.config reference-config-file: /etc/suricata/reference.config threshold-file: /etc/suricata/threshold.conf diff --git a/salt/suricata/suricata_mdengine.yaml b/salt/suricata/suricata_mdengine.yaml index d1fb7c2c3..1c3855501 100644 --- a/salt/suricata/suricata_mdengine.yaml +++ b/salt/suricata/suricata_mdengine.yaml @@ -70,8 +70,3 @@ suricata: - flow #- netflow #- metadata - rule-files: - - all.rules - - extraction.rules - - local.rules - - filters.rules From 230f5868f9ab59ae235d970d5319dd89276bdaab Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 15:14:27 -0400 Subject: [PATCH 12/56] sync sorules --- salt/idstools/sync_files.sls | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls index 64479e937..e8d5edda6 100644 --- a/salt/idstools/sync_files.sls +++ b/salt/idstools/sync_files.sls @@ -26,6 +26,13 @@ rulesdir: - group: 939 - makedirs: True +SOrulesdir: + file.directory: + - name: /opt/so/rules/nids/sorules + - user: 939 + - group: 939 + - makedirs: True + # Don't show changes because all.rules can be large synclocalnidsrules: file.recurse: @@ -35,3 +42,13 @@ synclocalnidsrules: - group: 939 - show_changes: False - include_pat: 'E@.rules' + +# Don't show changes because all.rules can be large +syncnidsSOrules: + file.recurse: + - name: /opt/so/rules/nids/sorules + - source: salt://idstools/sorules/ + - user: 939 + - group: 939 + - show_changes: False + - include_pat: 'E@.rules' From 5c704d7e5864dc16c37224d000c019b257315a5c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 15:20:44 -0400 Subject: [PATCH 13/56] run so-rule-update if idstools configs change --- salt/idstools/sync_files.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls index e8d5edda6..cc6c45baa 100644 --- a/salt/idstools/sync_files.sls +++ b/salt/idstools/sync_files.sls @@ -19,6 +19,12 @@ idstoolsetcsync: - group: 939 - template: jinja +run_so-rule-update: + cmd.run: + - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1' + - onchanges: + - file: idstoolsetcsync + rulesdir: file.directory: - name: /opt/so/rules/nids From 21c80e4953a4de0b0cfa58464808fc44e8665704 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 15:27:23 -0400 Subject: [PATCH 14/56] run so-rule-update after idstools container restart --- salt/idstools/enabled.sls | 8 ++++++++ salt/idstools/sync_files.sls | 6 ------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls index bf5650773..966cb6786 100644 --- a/salt/idstools/enabled.sls +++ b/salt/idstools/enabled.sls @@ -69,6 +69,14 @@ so-rule-update: - minute: '1' - hour: '7' +run_so-rule-update: + cmd.run: + - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1' + - require: + - docker_container: so-idstools + - onchanges: + - file: idstoolsetcsync + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls index cc6c45baa..e8d5edda6 100644 --- a/salt/idstools/sync_files.sls +++ b/salt/idstools/sync_files.sls @@ -19,12 +19,6 @@ idstoolsetcsync: - group: 939 - template: jinja -run_so-rule-update: - cmd.run: - - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1' - - onchanges: - - file: idstoolsetcsync - rulesdir: file.directory: - name: /opt/so/rules/nids From 3d4fd08547a32d713480e8e48f04e39fe6216182 Mon Sep 17 00:00:00 2001 From: bryant-treacle Date: Tue, 8 Aug 2023 15:28:06 -0400 Subject: [PATCH 15/56] Update defaults.yaml --- salt/soc/defaults.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index cb7d400a0..f97089e02 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -570,14 +570,13 @@ soc: - destination.geo.country_iso_code - user.name - source.ip - ':windows.sysmon_operational:': + '::sysmon_operational': - soc_timestamp - event.action - - process.executable + - winlog.computer_name - user.name - - file.target - - dns.question.name - - winlog.event_data.TargetObject + - process.executable + - process.pid '::network_connection': - soc_timestamp - source.ip From 036b81707b275f96f9ed13c0021be6ee765d690b Mon Sep 17 00:00:00 2001 From: bryant-treacle Date: Tue, 8 Aug 2023 16:10:54 -0400 Subject: [PATCH 16/56] Update defaults.yaml --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index f97089e02..49be076c0 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -69,7 +69,7 @@ soc: - log.id.uid - network.community_id - event.dataset - ':kratos:kratos.audit': + ':kratos:audit': - soc_timestamp - http_request.headers.x-real-ip - identity_id From 2f74b69cc39f1ed577bc48ce0df5025f9fcd58bf Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 8 Aug 2023 16:27:11 -0400 Subject: [PATCH 17/56] Update soup for 2.4.10 --- salt/manager/tools/sbin/soup | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d31ee997b..f8221a4f4 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -393,6 +393,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.2 ]] && up_to_2.4.3 [[ "$INSTALLEDVERSION" == 2.4.3 ]] && up_to_2.4.4 [[ "$INSTALLEDVERSION" == 2.4.4 ]] && up_to_2.4.5 + [[ "$INSTALLEDVERSION" == 2.4.5 ]] && up_to_2.4.10 true } @@ -403,6 +404,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.2 ]] && post_to_2.4.3 [[ "$POSTVERSION" == 2.4.3 ]] && post_to_2.4.4 [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5 + [[ "$POSTVERSION" == 2.4.5 ]] && post_to_2.4.10 true } @@ -422,6 +424,11 @@ post_to_2.4.5() { POSTVERSION=2.4.5 } +post_to_2.4.10() { + echo "Nothing to apply" + POSTVERSION=2.4.10 +} + stop_salt_master() { # kill all salt jobs across the grid because the hang indefinitely if they are queued and salt-master restarts set +e @@ -482,6 +489,12 @@ up_to_2.4.5() { INSTALLEDVERSION=2.4.5 } +up_to_2.4.10() { + echo "Nothing to do for 2.4.10" + + INSTALLEDVERSION=2.4.10 +} + determine_elastic_agent_upgrade() { if [[ $is_airgap -eq 0 ]]; then update_elastic_agent_airgap From 2dbe6798498d3086298b79ca355940ed98da96ee Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 17:05:03 -0400 Subject: [PATCH 18/56] force restart of filecheck if the config changes --- salt/strelka/filestream/config.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index a215967ee..d4615b174 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -85,6 +85,7 @@ filecheck_restart: - success_retcodes: [0,1] - onchanges: - file: filecheck_script + - file: filecheck_conf filecheck_run: cron.present: From 6da2f117f215ee856fe6800ff91c94ff11cea168 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 17:25:08 -0400 Subject: [PATCH 19/56] change which user runs filecheck cron based on md engine --- salt/strelka/filestream/config.sls | 30 ++++++++++++++++++++++++++---- salt/strelka/map.jinja | 2 -- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index d4615b174..9c0ef1357 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -6,7 +6,7 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} {% from 'strelka/map.jinja' import STRELKAMERGED %} -{% from 'strelka/map.jinja' import filecheck_runas %} +{% from 'vars/globals.map.jinja' import GLOBALS %} include: - strelka.config @@ -87,11 +87,33 @@ filecheck_restart: - file: filecheck_script - file: filecheck_conf -filecheck_run: +{% if GLOBALS.md_engine == 'ZEEK' %} + +filecheck_run_socore: cron.present: - name: 'ps -ef | grep filecheck | grep -v grep > /dev/null 2>&1 || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &' - - identifier: filecheck_run - - user: {{ filecheck_runas }} + - identifier: filecheck_run_socore + - user: socore + +remove_filecheck_run_suricata: + cron.absent: + - identifier: filecheck_run_suricata + - user: suricata + +{% elif GLOBALS.md_engine == 'SURICATA'%} + +filecheck_run_suricata: + cron.present: + - name: 'ps -ef | grep filecheck | grep -v grep > /dev/null 2>&1 || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &' + - identifier: filecheck_run_suricata + - user: suricata + +remove_filecheck_run_socore: + cron.absent: + - identifier: filecheck_run_socore + - user: socore + +{% endif %} filcheck_history_clean: cron.present: diff --git a/salt/strelka/map.jinja b/salt/strelka/map.jinja index 646f7a746..387036248 100644 --- a/salt/strelka/map.jinja +++ b/salt/strelka/map.jinja @@ -24,10 +24,8 @@ {% if GLOBALS.md_engine == "SURICATA" %} {% set extract_path = '/nsm/suricata/extracted' %} -{% set filecheck_runas = 'suricata' %} {% else %} {% set extract_path = '/nsm/zeek/extracted/complete' %} -{% set filecheck_runas = 'socore' %} {% endif %} {% do STRELKADEFAULTS.strelka.filecheck.update({'extract_path': extract_path}) %} From 553b758c61e87909f704b9159c13d30022ea3ac4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 17:28:14 -0400 Subject: [PATCH 20/56] update cronjobs first, the kill filecheck --- salt/strelka/filestream/config.sls | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index 9c0ef1357..193241f32 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -78,15 +78,6 @@ filecheck_script: - group: 939 - mode: 755 -filecheck_restart: - cmd.run: - - name: pkill -f "python3 /opt/so/conf/strelka/filecheck" - - hide_output: True - - success_retcodes: [0,1] - - onchanges: - - file: filecheck_script - - file: filecheck_conf - {% if GLOBALS.md_engine == 'ZEEK' %} filecheck_run_socore: @@ -115,6 +106,15 @@ remove_filecheck_run_socore: {% endif %} +filecheck_restart: + cmd.run: + - name: pkill -f "python3 /opt/so/conf/strelka/filecheck" + - hide_output: True + - success_retcodes: [0,1] + - onchanges: + - file: filecheck_script + - file: filecheck_conf + filcheck_history_clean: cron.present: - name: '/usr/bin/find /nsm/strelka/history/ -type f -mtime +2 -exec rm {} + > /dev/null 2>&1' From 58fe25623b6ece773278c293727428059e8944fb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 17:48:34 -0400 Subject: [PATCH 21/56] ensure ownership of /opt/so/log/strelka/filecheck_stdout.log --- salt/strelka/filestream/config.sls | 7 +++++++ salt/strelka/map.jinja | 2 ++ 2 files changed, 9 insertions(+) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index 193241f32..c827ff5fb 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -7,6 +7,7 @@ {% if sls.split('.')[0] in allowed_states %} {% from 'strelka/map.jinja' import STRELKAMERGED %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'strelka/map.jinja' import filecheck_runas %} include: - strelka.config @@ -78,6 +79,12 @@ filecheck_script: - group: 939 - mode: 755 +filecheck_stdout.log: + file.managed: + - name: /opt/so/log/strelka/filecheck_stdout.log + - user: {{ filecheck_runas }} + - group: {{ filecheck_runas }} + {% if GLOBALS.md_engine == 'ZEEK' %} filecheck_run_socore: diff --git a/salt/strelka/map.jinja b/salt/strelka/map.jinja index 387036248..646f7a746 100644 --- a/salt/strelka/map.jinja +++ b/salt/strelka/map.jinja @@ -24,8 +24,10 @@ {% if GLOBALS.md_engine == "SURICATA" %} {% set extract_path = '/nsm/suricata/extracted' %} +{% set filecheck_runas = 'suricata' %} {% else %} {% set extract_path = '/nsm/zeek/extracted/complete' %} +{% set filecheck_runas = 'socore' %} {% endif %} {% do STRELKADEFAULTS.strelka.filecheck.update({'extract_path': extract_path}) %} From 789fff561efdbccd10497b7cf66e496cfeb543f8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 17:55:30 -0400 Subject: [PATCH 22/56] ensure ownership of /opt/so/log/strelka/filecheck.log --- salt/strelka/filestream/config.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index c827ff5fb..993a59650 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -79,6 +79,12 @@ filecheck_script: - group: 939 - mode: 755 +filecheck.log: + file.managed: + - name: /opt/so/log/strelka/filecheck.log + - user: {{ filecheck_runas }} + - group: {{ filecheck_runas }} + filecheck_stdout.log: file.managed: - name: /opt/so/log/strelka/filecheck_stdout.log From e1e535b009701a9adda2bdda06ceeeb52790746f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 8 Aug 2023 18:38:18 -0400 Subject: [PATCH 23/56] Retry if exit code is error --- salt/elasticfleet/enabled.sls | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index bb6410f2c..096610af2 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -22,6 +22,7 @@ include: so-elastic-fleet-auto-configure-logstash-outputs: cmd.run: - name: /usr/sbin/so-elastic-fleet-outputs-update + - retry: True {% endif %} # If enabled, automatically update Fleet Server URLs & ES Connection @@ -29,6 +30,7 @@ so-elastic-fleet-auto-configure-logstash-outputs: so-elastic-fleet-auto-configure-server-urls: cmd.run: - name: /usr/sbin/so-elastic-fleet-urls-update + - retry: True {% endif %} # Automatically update Fleet Server Elasticsearch URLs @@ -36,6 +38,7 @@ so-elastic-fleet-auto-configure-server-urls: so-elastic-fleet-auto-configure-elasticsearch-urls: cmd.run: - name: /usr/sbin/so-elastic-fleet-es-url-update + - retry: True {% endif %} {% if SERVICETOKEN != '' %} From 00efc2f88f5bfdb28bf4dfb18df8855709486bfb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 07:31:31 -0400 Subject: [PATCH 24/56] rename workstation to desktop for firewall --- salt/firewall/defaults.yaml | 16 ++++++++-------- salt/firewall/soc_firewall.yaml | 18 +++++++++--------- salt/manager/tools/sbin/so-firewall-minion | 4 ++-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 9b8325a34..347ddd4b0 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -26,7 +26,7 @@ firewall: standalone: [] strelka_frontend: [] syslog: [] - workstation: [] + desktop: [] customhostgroup0: [] customhostgroup1: [] customhostgroup2: [] @@ -462,7 +462,7 @@ firewall: endgame: portgroups: - endgame - workstation: + desktop: portgroups: - yum customhostgroup0: @@ -514,7 +514,7 @@ firewall: receiver: portgroups: - salt_manager - workstation: + desktop: portgroups: - salt_manager self: @@ -650,7 +650,7 @@ firewall: endgame: portgroups: - endgame - workstation: + desktop: portgroups: - yum customhostgroup0: @@ -702,7 +702,7 @@ firewall: receiver: portgroups: - salt_manager - workstation: + desktop: portgroups: - salt_manager self: @@ -846,7 +846,7 @@ firewall: strelka_frontend: portgroups: - strelka_frontend - workstation: + desktop: portgroups: - yum customhostgroup0: @@ -901,7 +901,7 @@ firewall: receiver: portgroups: - salt_manager - workstation: + desktop: portgroups: - salt_manager self: @@ -1200,7 +1200,7 @@ firewall: analyst: portgroups: - nginx - workstation: + desktop: portgroups: - yum customhostgroup0: diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 8f8dbb69d..6ba5bea76 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -45,7 +45,7 @@ firewall: standalone: *hostgroupsettings strelka_frontend: *hostgroupsettings syslog: *hostgroupsettings - workstation: *hostgroupsettings + desktop: *hostgroupsettings customhostgroup0: &customhostgroupsettings description: List of IP or CIDR blocks to allow to this hostgroup. forcedType: "[]string" @@ -216,7 +216,7 @@ firewall: portgroups: *portgroupsdocker analyst: portgroups: *portgroupsdocker - workstation: + desktop: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -366,7 +366,7 @@ firewall: portgroups: *portgroupsdocker analyst: portgroups: *portgroupsdocker - workstation: + desktop: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -404,7 +404,7 @@ firewall: portgroups: *portgroupshost heavynode: portgroups: *portgroupshost - workstation: + desktop: portgroups: *portgroupshost customhostgroup0: portgroups: *portgroupshost @@ -457,7 +457,7 @@ firewall: portgroups: *portgroupsdocker analyst: portgroups: *portgroupsdocker - workstation: + desktop: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -495,7 +495,7 @@ firewall: portgroups: *portgroupshost heavynode: portgroups: *portgroupshost - workstation: + desktop: portgroups: *portgroupshost customhostgroup0: portgroups: *portgroupshost @@ -554,7 +554,7 @@ firewall: portgroups: *portgroupsdocker analyst: portgroups: *portgroupsdocker - workstation: + desktop: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -596,7 +596,7 @@ firewall: portgroups: *portgroupshost heavynode: portgroups: *portgroupshost - workstation: + desktop: portgroups: *portgroupshost customhostgroup0: portgroups: *portgroupshost @@ -822,7 +822,7 @@ firewall: portgroups: *portgroupsdocker analyst: portgroups: *portgroupsdocker - workstation: + desktop: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker diff --git a/salt/manager/tools/sbin/so-firewall-minion b/salt/manager/tools/sbin/so-firewall-minion index d3bbb3eeb..66a0afcea 100755 --- a/salt/manager/tools/sbin/so-firewall-minion +++ b/salt/manager/tools/sbin/so-firewall-minion @@ -79,7 +79,7 @@ fi 'RECEIVER') so-firewall includehost receiver "$IP" --apply ;; - 'WORKSTATION') - so-firewall includehost workstation "$IP" --apply + 'DESKTOP') + so-firewall includehost desktop "$IP" --apply ;; esac From 1440c7255994f7b262643a01f3ae540a53aa46c0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 08:06:51 -0400 Subject: [PATCH 25/56] changes for desktop referencing Rocky/CentOS to OEL --- salt/common/tools/sbin_jinja/so-desktop-install | 10 +++++----- salt/desktop/trusted-ca.sls | 2 +- salt/top.sls | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-desktop-install b/salt/common/tools/sbin_jinja/so-desktop-install index 448421f8e..2eb5c209f 100755 --- a/salt/common/tools/sbin_jinja/so-desktop-install +++ b/salt/common/tools/sbin_jinja/so-desktop-install @@ -6,8 +6,8 @@ # Elastic License 2.0. -{# we only want the script to install the desktop if it is Rocky -#} -{% if grains.os == 'Rocky' -%} +{# we only want the script to install the desktop if it is OEL -#} +{% if grains.os == 'OEL' -%} {# if this is a manager -#} {% if grains.master == grains.id.split('_')|first -%} @@ -80,12 +80,12 @@ echo "Since this is not a manager, the pillar values to enable Security Onion De {#- endif if this is a manager #} {% endif -%} -{#- if not Rocky #} +{#- if not OEL #} {%- else %} -echo "The Security Onion Desktop can only be installed on Rocky Linux. Please view the documentation at $doc_desktop_url." +echo "The Security Onion Desktop can only be installed on Oracle Linux. Please view the documentation at $doc_desktop_url." -{#- endif grains.os == Rocky #} +{#- endif grains.os == OEL #} {% endif -%} exit 0 diff --git a/salt/desktop/trusted-ca.sls b/salt/desktop/trusted-ca.sls index b9bde5ae5..87fc70ef9 100644 --- a/salt/desktop/trusted-ca.sls +++ b/salt/desktop/trusted-ca.sls @@ -31,6 +31,6 @@ update_ca_certs: desktop_trusted-ca_os_fail: test.fail_without_changes: - - comment: 'SO Desktop can only be installed on CentOS' + - comment: 'SO Desktop can only be installed on Oracle Linux' {% endif %} diff --git a/salt/top.sls b/salt/top.sls index bc51c2db1..2323731a1 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -277,10 +277,10 @@ base: - schedule - docker_clean - 'J@desktop:gui:enabled:^[Tt][Rr][Uu][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:Rocky )': + 'J@desktop:gui:enabled:^[Tt][Rr][Uu][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:OEL )': - match: compound - desktop - 'J@desktop:gui:enabled:^[Ff][Aa][Ll][Ss][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:Rocky )': + 'J@desktop:gui:enabled:^[Ff][Aa][Ll][Ss][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:OEL )': - match: compound - desktop.remove_gui From 4297d51a2db0b541f76f4f904bd0113cb94d7b89 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 9 Aug 2023 08:14:52 -0400 Subject: [PATCH 26/56] Refactor for multiple agents --- .../sbin_jinja/so-elastic-agent-grid-upgrade | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade new file mode 100644 index 000000000..23bf304f2 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade @@ -0,0 +1,38 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. + +. /usr/sbin/so-common + +# Only run on Managers +if ! is_manager_node; then + printf "Not a Manager Node... Exiting" + exit 0 +fi + +# Get current list of Grid Node Agents that need to be upgraded +RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=policy_id%20%3A%20so-grid-nodes_%2A&showInactive=false&showUpgradeable=true&getStatusSummary=true") + +# Check to make sure that the server responded with good data - else, bail from script +CHECKSUM=$(jq -r '.statusSummary.online' <<< "$RAW_JSON") +if [ "$CHECKSUM" -lt 1 ]; then + printf "Failed to query for current Grid Agents...\n" + exit 1 +fi + +# Generate list of Node Agents that need updates +OUTDATED_LIST=$(jq -r '.items | map(.id) | (tojson)' <<< "$RAW_JSON") + +if [ "$OUTDATED_LIST" != '[]' ]; then + AGENTNUMBERS=$(jq -r '.total' <<< "$RAW_JSON") + printf "Initiating upgrades for $AGENTNUMBERS Agents to Elastic $ELASTIC_AGENT_TARBALL_VERSION...\n\n" + + # Generate updated JSON payload + JSON_STRING=$(jq -n --arg ELASTICVERSION $ELASTIC_AGENT_TARBALL_VERSION --arg UPDATELIST $OUTDATED_LIST '{"version": $ELASTICVERSION,"agents": $UPDATELIST }') + + # Update Node Agents + curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "http://localhost:5601/api/fleet/agents/bulk_upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +else + printf "No Agents need updates... Exiting\n\n" + exit 0 +fi \ No newline at end of file From 2d25e352d4fe201d442b82cf233d412a5ad6258d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 08:18:13 -0400 Subject: [PATCH 27/56] write to adv_ pillar file since that is where it would be stored from using the soc ui --- salt/common/tools/sbin_jinja/so-desktop-install | 2 +- salt/desktop/packages.sls | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-desktop-install b/salt/common/tools/sbin_jinja/so-desktop-install index 2eb5c209f..06385e810 100755 --- a/salt/common/tools/sbin_jinja/so-desktop-install +++ b/salt/common/tools/sbin_jinja/so-desktop-install @@ -13,7 +13,7 @@ source /usr/sbin/so-common doc_desktop_url="$DOC_BASE_URL/desktop.html" -pillar_file="/opt/so/saltstack/local/pillar/minions/{{grains.id}}.sls" +pillar_file="/opt/so/saltstack/local/pillar/minions/adv_{{grains.id}}.sls" if [ -f "$pillar_file" ]; then if ! grep -q "^desktop:$" "$pillar_file"; then diff --git a/salt/desktop/packages.sls b/salt/desktop/packages.sls index 5c0121e7b..524c2c266 100644 --- a/salt/desktop/packages.sls +++ b/salt/desktop/packages.sls @@ -3,7 +3,6 @@ {# we only want this state to run it is CentOS #} {% if GLOBALS.os == 'OEL' %} - desktop_packages: pkg.installed: - pkgs: From e586d6b96755110a122323577c3b4395d5add033 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 9 Aug 2023 08:30:19 -0400 Subject: [PATCH 28/56] Extract Elastic Agent tarball for airgap soup --- salt/manager/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d31ee997b..b7abd05d8 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -492,6 +492,7 @@ determine_elastic_agent_upgrade() { update_elastic_agent_airgap() { rsync -av /tmp/soagupdate/fleet/* /nsm/elastic-fleet/artifacts/ + tar -xf "$ELASTIC_AGENT_FILE" -C "$ELASTIC_AGENT_EXPANSION_DIR" } verify_upgradespace() { From fe7a940082004c178995c18e1aad749ffd8e8331 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 08:31:54 -0400 Subject: [PATCH 29/56] add details for enabling in soc gui --- salt/common/tools/sbin_jinja/so-desktop-install | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-desktop-install b/salt/common/tools/sbin_jinja/so-desktop-install index 06385e810..bd3d9b373 100755 --- a/salt/common/tools/sbin_jinja/so-desktop-install +++ b/salt/common/tools/sbin_jinja/so-desktop-install @@ -65,7 +65,7 @@ if [ -f "$pillar_file" ]; then fi else # desktop is already added echo "The desktop pillar already exists in $pillar_file." - echo "To enable/disable the gui, set 'desktop:gui:enabled' to true or false in $pillar_file." + echo "To enable/disable the gui, set 'desktop:gui:enabled' to true or false in $pillar_file. Alternatively, this can be set in the SOC UI under advanced." echo "Additional documentation can be found at $doc_desktop_url." fi else # if the pillar file doesn't exist @@ -75,7 +75,12 @@ fi {#- if this is not a manager #} {% else -%} -echo "Since this is not a manager, the pillar values to enable Security Onion Desktop must be set manually. Please view the documentation at $doc_desktop_url." +echo "Since this is not a manager, the pillar values to enable Security Onion Desktop must be set manually. This can be enabled in the SOC UI under advanced by adding the following:" +echo "desktop:" +echo " gui:" +echo " enabled: true" +echo "" +echo "Please view the documentation at $doc_desktop_url." {#- endif if this is a manager #} {% endif -%} From 6413050f2e27fd004dfcfd7b2925b5133fa044e9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 08:39:46 -0400 Subject: [PATCH 30/56] set doc_desktop_url before jinja --- salt/common/tools/sbin_jinja/so-desktop-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-desktop-install b/salt/common/tools/sbin_jinja/so-desktop-install index bd3d9b373..6275bb3b6 100755 --- a/salt/common/tools/sbin_jinja/so-desktop-install +++ b/salt/common/tools/sbin_jinja/so-desktop-install @@ -5,14 +5,14 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +source /usr/sbin/so-common +doc_desktop_url="$DOC_BASE_URL/desktop.html" {# we only want the script to install the desktop if it is OEL -#} {% if grains.os == 'OEL' -%} {# if this is a manager -#} {% if grains.master == grains.id.split('_')|first -%} -source /usr/sbin/so-common -doc_desktop_url="$DOC_BASE_URL/desktop.html" pillar_file="/opt/so/saltstack/local/pillar/minions/adv_{{grains.id}}.sls" if [ -f "$pillar_file" ]; then From a443c654e58f507bade98d12812ca5969985ffc6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 08:48:00 -0400 Subject: [PATCH 31/56] fix desktop pillar in setup --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 0f73a11a6..d138d97df 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -117,7 +117,7 @@ desktop_pillar() { " mainint: '$MNIC'"\ "desktop:"\ " gui:"\ - " enabled: true" >> "$pillar_file"\ + " enabled: true"\ "sensoroni:"\ " config:"\ " node_description: '${NODE_DESCRIPTION//\'/''}'" > $pillar_file From 28dfdbf06dc6de143716e94fd9c3432799e1421f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 08:51:39 -0400 Subject: [PATCH 32/56] securityonion_desktop is just desktop --- salt/firewall/defaults.yaml | 1 - salt/firewall/soc_firewall.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 347ddd4b0..ff127c419 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -20,7 +20,6 @@ firewall: managersearch: [] receiver: [] searchnode: [] - securityonion_desktop: [] self: [] sensor: [] standalone: [] diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 6ba5bea76..209484b6e 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -39,7 +39,6 @@ firewall: managersearch: *hostgroupsettings receiver: *hostgroupsettings searchnode: *hostgroupsettings - securityonion_desktop: *hostgroupsettings self: *ROhostgroupsettingsadv sensor: *hostgroupsettings standalone: *hostgroupsettings From bf78faa0f081c873d371fa0ec2cf22c970755add Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 9 Aug 2023 10:43:34 -0400 Subject: [PATCH 33/56] Enable upgrade check during state run --- salt/elasticfleet/enabled.sls | 5 +++++ .../tools/sbin_jinja/so-elastic-agent-grid-upgrade | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 096610af2..82c7735db 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -109,6 +109,11 @@ so-elastic-fleet: so-elastic-fleet-integrations: cmd.run: - name: /usr/sbin/so-elastic-fleet-integration-policy-load + +so-elastic-agent-grid-upgrade: + cmd.run: + - name: /usr/sbin/so-elastic-agent-grid-upgrade + - retry: True {% endif %} delete_so-elastic-fleet_so-status.disabled: diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade index 23bf304f2..b1ca8c476 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade @@ -14,8 +14,8 @@ fi RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=policy_id%20%3A%20so-grid-nodes_%2A&showInactive=false&showUpgradeable=true&getStatusSummary=true") # Check to make sure that the server responded with good data - else, bail from script -CHECKSUM=$(jq -r '.statusSummary.online' <<< "$RAW_JSON") -if [ "$CHECKSUM" -lt 1 ]; then +CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON") +if [ "$CHECKSUM" -ne 1 ]; then printf "Failed to query for current Grid Agents...\n" exit 1 fi From 8844e305ab5da068c670061a45c0382f500ee40d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 11:18:47 -0400 Subject: [PATCH 34/56] use sensor.interface for suricata. make af-packet.interface ro in soc ui --- salt/suricata/map.jinja | 2 +- salt/suricata/soc_suricata.yaml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 5576117cc..2d79c30fd 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -11,7 +11,7 @@ {# suricata.config.af-packet has to be rewritten here since we cant display '- interface' in the ui #} {# we are limited to only one iterface #} {% load_yaml as afpacket %} -- interface: {{ SURICATAMERGED.config['af-packet'].interface }} +- interface: {{ GLOBALS.sensor.interface) }} cluster-id: {{ SURICATAMERGED.config['af-packet']['cluster-id'] }} cluster-type: {{ SURICATAMERGED.config['af-packet']['cluster-type'] }} defrag: {{ SURICATAMERGED.config['af-packet'].defrag }} diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index f13e89618..30f277c0a 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -14,7 +14,9 @@ suricata: config: af-packet: interface: - description: The network interface that Suricata will monitor. + description: The network interface that Suricata will monitor. This is set under sensor > interface. + advanced: True + readonly: True helpLink: suricata.html cluster-id: advanced: True From 30e3fbb41c86fafacad01849c502d2bd9cae1753 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 11:21:16 -0400 Subject: [PATCH 35/56] remove extra ) --- salt/suricata/map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 2d79c30fd..01d019de8 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -11,7 +11,7 @@ {# suricata.config.af-packet has to be rewritten here since we cant display '- interface' in the ui #} {# we are limited to only one iterface #} {% load_yaml as afpacket %} -- interface: {{ GLOBALS.sensor.interface) }} +- interface: {{ GLOBALS.sensor.interface }} cluster-id: {{ SURICATAMERGED.config['af-packet']['cluster-id'] }} cluster-type: {{ SURICATAMERGED.config['af-packet']['cluster-type'] }} defrag: {{ SURICATAMERGED.config['af-packet'].defrag }} From dfe916d7c8a996c5070fc89fb1ade0d957480bfd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 15:19:17 -0400 Subject: [PATCH 36/56] add annotation for so-logs index --- salt/elasticsearch/soc_elasticsearch.yaml | 109 ++++++++++++++++------ 1 file changed, 78 insertions(+), 31 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index da22268f6..2228eccf6 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -46,28 +46,26 @@ elasticsearch: description: Max number of boolean clauses per query. global: True helpLink: elasticsearch.html - index_settings: - so-elasticsearch: &indexSettings - warm: - description: Age (in days) of this index before it will move to warm storage, if warm nodes are present. Once moved, events on this index can take longer to fetch. - global: True - helpLink: elasticsearch.html - close: - description: Age (in days) of this index before it will be closed. Once closed, events on this index cannot be retrieved without first re-opening the index. - global: True - helpLink: elasticsearch.html - delete: - description: Age (in days) of this index before it will be deleted. Once deleted, events are permanently unrecoverable. - global: True - helpLink: elasticsearch.html + index_settings: + so-logs: &indexSettings index_sorting: description: Sorts the index by event time, at the cost of additional processing resource consumption. global: True helpLink: elasticsearch.html index_template: + index_patterns: + description: Patterns for matching multiple indices or tables. + forceType: "[]string" + multiline: True + global: True + helpLink: elasticsearch.html template: settings: index: + number_of_replicas: + description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. + global: True + helpLink: elasticsearch.html mapping: total_fields: limit: @@ -75,17 +73,59 @@ elasticsearch: global: True helpLink: elasticsearch.html refresh_interval: - description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation. - global: True - helpLink: elasticsearch.html + description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation. + global: True + helpLink: elasticsearch.html number_of_shards: - description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs. + description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs. + global: True + helpLink: elasticsearch.html + sort: + field: + description: The field to sort by. Must set index_sorting to True. global: True helpLink: elasticsearch.html - number_of_replicas: - description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. + order: + description: The order to sort by. Must set index_sorting to True. global: True helpLink: elasticsearch.html + mappings: + _meta: + package: + name: + description: Meta settings for the mapping. + global: True + helpLink: elasticsearch.html + managed_by: + description: Meta settings for the mapping. + global: True + helpLink: elasticsearch.html + managed: + description: Meta settings for the mapping. + forcedType: bool + global: True + helpLink: elasticsearch.html + composed_of: + description: The index template is composed of these component templates. + forcedType: "[]string" + global: True + helpLink: elasticsearch.html + priority: + description: The priority of the index template. + forcedType: int + global: True + helpLink: elasticsearch.html + data_stream: + hidden: + description: Hide the data stream. + forcedType: bool + global: True + helpLink: elasticsearch.html + allow_custom_routing: + description: Allow custom routing for the data stream. + forcedType: bool + global: True + helpLink: elasticsearch.html policy: phases: hot: @@ -97,6 +137,7 @@ elasticsearch: set_priority: priority: description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + forcedType: int global: True helpLink: elasticsearch.html rollover: @@ -117,20 +158,26 @@ elasticsearch: set_priority: priority: description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + forcedType: int global: True helpLink: elasticsearch.html delete: min_age: description: Minimum age of index. This determines when the index should be deleted. global: True - helpLink: elastic - so-endgame: *indexSettings - so-firewall: *indexSettings - so-import: *indexSettings - so-kibana: *indexSettings - so-logstash: *indexSettings - so-osquery: *indexSettings - so-redis: *indexSettings - so-strelka: *indexSettings - so-syslog: *indexSettings - so-zeek: *indexSettings + helpLink: elasticsearch.html + _meta: + package: + name: + description: Meta settings for the mapping. + global: True + helpLink: elasticsearch.html + managed_by: + description: Meta settings for the mapping. + global: True + helpLink: elasticsearch.html + managed: + description: Meta settings for the mapping. + forcedType: bool + global: True + helpLink: elasticsearch.html From f9e272dd8f07613c748fee6540c88e4bea59b145 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 16:09:23 -0400 Subject: [PATCH 37/56] add additional annotations for elasticsearch index settings --- salt/elasticsearch/soc_elasticsearch.yaml | 86 +++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 2228eccf6..89d347b42 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -181,3 +181,89 @@ elasticsearch: forcedType: bool global: True helpLink: elasticsearch.html + so-logs-system.auth: *indexSettings + so-logs-system.syslog: *indexSettings + so-logs-system.system: *indexSettings + so-logs-system.application: *indexSettings + so-logs-system.security: *indexSettings + so-logs-windows.forwarded: *indexSettings + so-logs-windows.powershell: *indexSettings + so-logs-windows.powershell_operational: *indexSettings + so-logs-windows.sysmon_operational: *indexSettings + so-logs-aws.cloudtrail: *indexSettings + so-logs-aws.cloudwatch_logs: *indexSettings + so-logs-aws.ec2_logs: *indexSettings + so-logs-aws.elb_logs: *indexSettings + so-logs-aws.firewall_logs: *indexSettings + so-logs-aws.route53_public_logs: *indexSettings + so-logs-aws.route53_resolver_logs: *indexSettings + so-logs-aws.s3access: *indexSettings + so-logs-aws.vpcflow: *indexSettings + so-logs-aws.waf: *indexSettings + so-logs-azure.activitylogs: *indexSettings + so-logs-azure.application_gateway: *indexSettings + so-logs-azure.auditlogs: *indexSettings + so-logs-azure.eventhub: *indexSettings + so-logs-azure.firewall_logs: *indexSettings + so-logs-azure.identity_protection: *indexSettings + so-logs-azure.platformlogs: *indexSettings + so-logs-azure.provisioning: *indexSettings + so-logs-azure.signinlogs: *indexSettings + so-logs-azure.springcloudlogs: *indexSettings + so-logs-cloudflare.audit: *indexSettings + so-logs-cloudflare.logpull: *indexSettings + so-logs-fim.event: *indexSettings + so-logs-github.audit: *indexSettings + so-logs-github.code_scanning: *indexSettings + so-logs-github.dependabot: *indexSettings + so-logs-github.issues: *indexSettings + so-logs-github.secret_scanning: *indexSettings + so-logs-google_workspace.access_transparency: *indexSettings + so-logs-google_workspace.admin: *indexSettings + so-logs-google_workspace.alert: *indexSettings + so-logs-google_workspace.context_aware_access: *indexSettings + so-logs-google_workspace.device: *indexSettings + so-logs-google_workspace.drive: *indexSettings + so-logs-google_workspace.gcp: *indexSettings + so-logs-google_workspace.group_enterprise: *indexSettings + so-logs-google_workspace.groups: *indexSettings + so-logs-google_workspace.login: *indexSettings + so-logs-google_workspace.rules: *indexSettings + so-logs-google_workspace.saml: *indexSettings + so-logs-google_workspace.token: *indexSettings + so-logs-google_workspace.user_accounts: *indexSettings + so-logs-1password.item_usages: *indexSettings + so-logs-1password.signin_attempts: *indexSettings + so-logs-osquery-manager-actions: *indexSettings + so-logs-osquery-manager-action.responses: *indexSettings + so-logs-elastic_agent.apm_server: *indexSettings + so-logs-elastic_agent.auditbeat: *indexSettings + so-logs-elastic_agent.cloudbeat: *indexSettings + so-logs-elastic_agent.endpoint_security: *indexSettings + so-logs-endpoint.alerts: *indexSettings + so-logs-endpoint.events.api: *indexSettings + so-logs-endpoint.events.file: *indexSettings + so-logs-endpoint.events.library: *indexSettings + so-logs-endpoint.events.network: *indexSettings + so-logs-endpoint.events.process: *indexSettings + so-logs-endpoint.events.registry: *indexSettings + so-logs-endpoint.events.security: *indexSettings + so-logs-elastic_agent.filebeat: *indexSettings + so-logs-elastic_agent.fleet_server: *indexSettings + so-logs-elastic_agent.heartbeat: *indexSettings + so-logs-elastic_agent: *indexSettings + so-logs-elastic_agent.metricbeat: *indexSettings + so-logs-elastic_agent.osquerybeat: *indexSettings + so-logs-elastic_agent.packetbeat: *indexSettings + so-case: *indexSettings + so-common: *indexSettings + so-endgame: *indexSettings + so-idh: *indexSettings + so-suricata: *indexSettings + so-import: *indexSettings + so-kratos: *indexSettings + so-logstash: *indexSettings + so-redis: *indexSettings + so-strelka: *indexSettings + so-syslog: *indexSettings + so-zeek: *indexSettings From e844cf11db85ba2a9dd51c07ad72e3a1e673a911 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 9 Aug 2023 16:38:27 -0400 Subject: [PATCH 38/56] Move base_url to cert SAN --- salt/ssl/init.sls | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 063172e00..a25a7c270 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -153,8 +153,8 @@ etc_elasticfleet_crt: - ca_server: {{ ca_server }} - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-server.key - - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} + - CN: {{ GLOBALS.hostname }} + - subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True @@ -210,8 +210,8 @@ etc_elasticfleet_logstash_crt: - ca_server: {{ ca_server }} - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-logstash.key - - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} + - CN: {{ GLOBALS.hostname }} + - subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True From 874dab75355479338e7dd3a06a1ff3b75615a86b Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 9 Aug 2023 19:02:53 -0400 Subject: [PATCH 39/56] Unset defaults --- .../tools/sbin_jinja/so-elastic-fleet-es-url-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update index 975a8aec1..3b0e01f43 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update @@ -13,7 +13,7 @@ fi function update_es_urls() { # Generate updated JSON payload - JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}') + JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"config_yaml":""}') # Update Fleet Elasticsearch URLs curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" From 4d497022dbcd9730b2e0903d93a388bc48c7c564 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 10 Aug 2023 09:52:18 -0400 Subject: [PATCH 40/56] replace . with _x_ for soc ui compat --- salt/elasticsearch/defaults.yaml | 144 +++++++++--------- salt/elasticsearch/soc_elasticsearch.yaml | 144 +++++++++--------- salt/elasticsearch/template.map.jinja | 6 +- .../so-elasticsearch-ilm-policy-load | 3 +- 4 files changed, 149 insertions(+), 148 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 035079f54..579197040 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -113,7 +113,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-system.auth: + so-logs-system_x_auth: index_sorting: False index_template: index_patterns: @@ -132,7 +132,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-system.syslog: + so-logs-system_x_syslog: index_sorting: False index_template: index_patterns: @@ -151,7 +151,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-system.system: + so-logs-system_x_system: index_sorting: False index_template: index_patterns: @@ -170,7 +170,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-system.application: + so-logs-system_x_application: index_sorting: False index_template: index_patterns: @@ -189,7 +189,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-system.security: + so-logs-system_x_security: index_sorting: False index_template: index_patterns: @@ -208,7 +208,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-windows.forwarded: + so-logs-windows_x_forwarded: index_sorting: False index_template: index_patterns: @@ -226,7 +226,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-windows.powershell: + so-logs-windows_x_powershell: index_sorting: False index_template: index_patterns: @@ -244,7 +244,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-windows.powershell_operational: + so-logs-windows_x_powershell_operational: index_sorting: False index_template: index_patterns: @@ -262,7 +262,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-windows.sysmon_operational: + so-logs-windows_x_sysmon_operational: index_sorting: False index_template: index_patterns: @@ -280,7 +280,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.cloudtrail: + so-logs-aws_x_cloudtrail: index_sorting: False index_template: index_patterns: @@ -298,7 +298,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.cloudwatch_logs: + so-logs-aws_x_cloudwatch_logs: index_sorting: False index_template: index_patterns: @@ -316,7 +316,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.ec2_logs: + so-logs-aws_x_ec2_logs: index_sorting: False index_template: index_patterns: @@ -334,7 +334,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.elb_logs: + so-logs-aws_x_elb_logs: index_sorting: False index_template: index_patterns: @@ -352,7 +352,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.firewall_logs: + so-logs-aws_x_firewall_logs: index_sorting: False index_template: index_patterns: @@ -370,7 +370,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.route53_public_logs: + so-logs-aws_x_route53_public_logs: index_sorting: False index_template: index_patterns: @@ -388,7 +388,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.route53_resolver_logs: + so-logs-aws_x_route53_resolver_logs: index_sorting: False index_template: index_patterns: @@ -406,7 +406,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.s3access: + so-logs-aws_x_s3access: index_sorting: False index_template: index_patterns: @@ -424,7 +424,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.vpcflow: + so-logs-aws_x_vpcflow: index_sorting: False index_template: index_patterns: @@ -442,7 +442,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.waf: + so-logs-aws_x_waf: index_sorting: False index_template: index_patterns: @@ -460,7 +460,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.activitylogs: + so-logs-azure_x_activitylogs: index_sorting: False index_template: index_patterns: @@ -478,7 +478,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.application_gateway: + so-logs-azure_x_application_gateway: index_sorting: False index_template: index_patterns: @@ -496,7 +496,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.auditlogs: + so-logs-azure_x_auditlogs: index_sorting: False index_template: index_patterns: @@ -514,7 +514,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.eventhub: + so-logs-azure_x_eventhub: index_sorting: False index_template: index_patterns: @@ -532,7 +532,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.firewall_logs: + so-logs-azure_x_firewall_logs: index_sorting: False index_template: index_patterns: @@ -550,7 +550,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.identity_protection: + so-logs-azure_x_identity_protection: index_sorting: False index_template: index_patterns: @@ -568,7 +568,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.platformlogs: + so-logs-azure_x_platformlogs: index_sorting: False index_template: index_patterns: @@ -586,7 +586,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.provisioning: + so-logs-azure_x_provisioning: index_sorting: False index_template: index_patterns: @@ -604,7 +604,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.signinlogs: + so-logs-azure_x_signinlogs: index_sorting: False index_template: index_patterns: @@ -622,7 +622,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.springcloudlogs: + so-logs-azure_x_springcloudlogs: index_sorting: False index_template: index_patterns: @@ -640,7 +640,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-cloudflare.audit: + so-logs-cloudflare_x_audit: index_sorting: False index_template: index_patterns: @@ -658,7 +658,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-cloudflare.logpull: + so-logs-cloudflare_x_logpull: index_sorting: False index_template: index_patterns: @@ -676,7 +676,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-fim.event: + so-logs-fim_x_event: index_sorting: False index_template: index_patterns: @@ -694,7 +694,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-github.audit: + so-logs-github_x_audit: index_sorting: False index_template: index_patterns: @@ -712,7 +712,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-github.code_scanning: + so-logs-github_x_code_scanning: index_sorting: False index_template: index_patterns: @@ -730,7 +730,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-github.dependabot: + so-logs-github_x_dependabot: index_sorting: False index_template: index_patterns: @@ -748,7 +748,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-github.issues: + so-logs-github_x_issues: index_sorting: False index_template: index_patterns: @@ -766,7 +766,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-github.secret_scanning: + so-logs-github_x_secret_scanning: index_sorting: False index_template: index_patterns: @@ -784,7 +784,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.access_transparency: + so-logs-google_workspace_x_access_transparency: index_sorting: False index_template: index_patterns: @@ -802,7 +802,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.admin: + so-logs-google_workspace_x_admin: index_sorting: False index_template: index_patterns: @@ -820,7 +820,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.alert: + so-logs-google_workspace_x_alert: index_sorting: False index_template: index_patterns: @@ -838,7 +838,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.context_aware_access: + so-logs-google_workspace_x_context_aware_access: index_sorting: False index_template: index_patterns: @@ -856,7 +856,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.device: + so-logs-google_workspace_x_device: index_sorting: False index_template: index_patterns: @@ -874,7 +874,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.drive: + so-logs-google_workspace_x_drive: index_sorting: False index_template: index_patterns: @@ -892,7 +892,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.gcp: + so-logs-google_workspace_x_gcp: index_sorting: False index_template: index_patterns: @@ -910,7 +910,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.group_enterprise: + so-logs-google_workspace_x_group_enterprise: index_sorting: False index_template: index_patterns: @@ -928,7 +928,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.groups: + so-logs-google_workspace_x_groups: index_sorting: False index_template: index_patterns: @@ -946,7 +946,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.login: + so-logs-google_workspace_x_login: index_sorting: False index_template: index_patterns: @@ -964,7 +964,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.rules: + so-logs-google_workspace_x_rules: index_sorting: False index_template: index_patterns: @@ -982,7 +982,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.saml: + so-logs-google_workspace_x_saml: index_sorting: False index_template: index_patterns: @@ -1000,7 +1000,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.token: + so-logs-google_workspace_x_token: index_sorting: False index_template: index_patterns: @@ -1018,7 +1018,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.user_accounts: + so-logs-google_workspace_x_user_accounts: index_sorting: False index_template: index_patterns: @@ -1036,7 +1036,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-1password.item_usages: + so-logs-1password_x_item_usages: index_sorting: False index_template: index_patterns: @@ -1054,7 +1054,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-1password.signin_attempts: + so-logs-1password_x_signin_attempts: index_sorting: False index_template: index_patterns: @@ -1089,7 +1089,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-osquery-manager-action.responses: + so-logs-osquery-manager-action_x_responses: index_sorting: False index_template: index_patterns: @@ -1106,7 +1106,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.apm_server: + so-logs-elastic_agent_x_apm_server: index_sorting: False index_template: index_patterns: @@ -1160,7 +1160,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.auditbeat: + so-logs-elastic_agent_x_auditbeat: index_sorting: False index_template: index_patterns: @@ -1214,7 +1214,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.cloudbeat: + so-logs-elastic_agent_x_cloudbeat: index_sorting: False index_template: index_patterns: @@ -1265,7 +1265,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.endpoint_security: + so-logs-elastic_agent_x_endpoint_security: index_sorting: False index_template: index_patterns: @@ -1314,7 +1314,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.alerts: + so-logs-endpoint_x_alerts: index_sorting: False index_template: index_patterns: @@ -1363,7 +1363,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.api: + so-logs-endpoint_x_events_x_api: index_sorting: False index_template: index_patterns: @@ -1412,7 +1412,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.file: + so-logs-endpoint_x_events_x_file: index_sorting: False index_template: index_patterns: @@ -1461,7 +1461,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.library: + so-logs-endpoint_x_events_x_library: index_sorting: False index_template: index_patterns: @@ -1510,7 +1510,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.network: + so-logs-endpoint_x_events_x_network: index_sorting: False index_template: index_patterns: @@ -1559,7 +1559,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.process: + so-logs-endpoint_x_events_x_process: index_sorting: False index_template: index_patterns: @@ -1608,7 +1608,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.registry: + so-logs-endpoint_x_events_x_registry: index_sorting: False index_template: index_patterns: @@ -1657,7 +1657,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.security: + so-logs-endpoint_x_events_x_security: index_sorting: False index_template: index_patterns: @@ -1706,7 +1706,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.filebeat: + so-logs-elastic_agent_x_filebeat: index_sorting: False index_template: index_patterns: @@ -1755,7 +1755,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.fleet_server: + so-logs-elastic_agent_x_fleet_server: index_sorting: False index_template: index_patterns: @@ -1801,7 +1801,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.heartbeat: + so-logs-elastic_agent_x_heartbeat: index_sorting: False index_template: index_patterns: @@ -1907,7 +1907,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.metricbeat: + so-logs-elastic_agent_x_metricbeat: index_sorting: False index_template: index_patterns: @@ -1956,7 +1956,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.osquerybeat: + so-logs-elastic_agent_x_osquerybeat: index_sorting: False index_template: index_patterns: @@ -2005,7 +2005,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.packetbeat: + so-logs-elastic_agent_x_packetbeat: index_sorting: False index_template: index_patterns: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 89d347b42..889e9f6a4 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -181,80 +181,80 @@ elasticsearch: forcedType: bool global: True helpLink: elasticsearch.html - so-logs-system.auth: *indexSettings - so-logs-system.syslog: *indexSettings - so-logs-system.system: *indexSettings - so-logs-system.application: *indexSettings - so-logs-system.security: *indexSettings - so-logs-windows.forwarded: *indexSettings - so-logs-windows.powershell: *indexSettings - so-logs-windows.powershell_operational: *indexSettings - so-logs-windows.sysmon_operational: *indexSettings - so-logs-aws.cloudtrail: *indexSettings - so-logs-aws.cloudwatch_logs: *indexSettings - so-logs-aws.ec2_logs: *indexSettings - so-logs-aws.elb_logs: *indexSettings - so-logs-aws.firewall_logs: *indexSettings - so-logs-aws.route53_public_logs: *indexSettings - so-logs-aws.route53_resolver_logs: *indexSettings - so-logs-aws.s3access: *indexSettings - so-logs-aws.vpcflow: *indexSettings - so-logs-aws.waf: *indexSettings - so-logs-azure.activitylogs: *indexSettings - so-logs-azure.application_gateway: *indexSettings - so-logs-azure.auditlogs: *indexSettings - so-logs-azure.eventhub: *indexSettings - so-logs-azure.firewall_logs: *indexSettings - so-logs-azure.identity_protection: *indexSettings - so-logs-azure.platformlogs: *indexSettings - so-logs-azure.provisioning: *indexSettings - so-logs-azure.signinlogs: *indexSettings - so-logs-azure.springcloudlogs: *indexSettings - so-logs-cloudflare.audit: *indexSettings - so-logs-cloudflare.logpull: *indexSettings - so-logs-fim.event: *indexSettings - so-logs-github.audit: *indexSettings - so-logs-github.code_scanning: *indexSettings - so-logs-github.dependabot: *indexSettings - so-logs-github.issues: *indexSettings - so-logs-github.secret_scanning: *indexSettings - so-logs-google_workspace.access_transparency: *indexSettings - so-logs-google_workspace.admin: *indexSettings - so-logs-google_workspace.alert: *indexSettings - so-logs-google_workspace.context_aware_access: *indexSettings - so-logs-google_workspace.device: *indexSettings - so-logs-google_workspace.drive: *indexSettings - so-logs-google_workspace.gcp: *indexSettings - so-logs-google_workspace.group_enterprise: *indexSettings - so-logs-google_workspace.groups: *indexSettings - so-logs-google_workspace.login: *indexSettings - so-logs-google_workspace.rules: *indexSettings - so-logs-google_workspace.saml: *indexSettings - so-logs-google_workspace.token: *indexSettings - so-logs-google_workspace.user_accounts: *indexSettings - so-logs-1password.item_usages: *indexSettings - so-logs-1password.signin_attempts: *indexSettings + so-logs-system_x_auth: *indexSettings + so-logs-system_x_syslog: *indexSettings + so-logs-system_x_system: *indexSettings + so-logs-system_x_application: *indexSettings + so-logs-system_x_security: *indexSettings + so-logs-windows_x_forwarded: *indexSettings + so-logs-windows_x_powershell: *indexSettings + so-logs-windows_x_powershell_operational: *indexSettings + so-logs-windows_x_sysmon_operational: *indexSettings + so-logs-aws_x_cloudtrail: *indexSettings + so-logs-aws_x_cloudwatch_logs: *indexSettings + so-logs-aws_x_ec2_logs: *indexSettings + so-logs-aws_x_elb_logs: *indexSettings + so-logs-aws_x_firewall_logs: *indexSettings + so-logs-aws_x_route53_public_logs: *indexSettings + so-logs-aws_x_route53_resolver_logs: *indexSettings + so-logs-aws_x_s3access: *indexSettings + so-logs-aws_x_vpcflow: *indexSettings + so-logs-aws_x_waf: *indexSettings + so-logs-azure_x_activitylogs: *indexSettings + so-logs-azure_x_application_gateway: *indexSettings + so-logs-azure_x_auditlogs: *indexSettings + so-logs-azure_x_eventhub: *indexSettings + so-logs-azure_x_firewall_logs: *indexSettings + so-logs-azure_x_identity_protection: *indexSettings + so-logs-azure_x_platformlogs: *indexSettings + so-logs-azure_x_provisioning: *indexSettings + so-logs-azure_x_signinlogs: *indexSettings + so-logs-azure_x_springcloudlogs: *indexSettings + so-logs-cloudflare_x_audit: *indexSettings + so-logs-cloudflare_x_logpull: *indexSettings + so-logs-fim_x_event: *indexSettings + so-logs-github_x_audit: *indexSettings + so-logs-github_x_code_scanning: *indexSettings + so-logs-github_x_dependabot: *indexSettings + so-logs-github_x_issues: *indexSettings + so-logs-github_x_secret_scanning: *indexSettings + so-logs-google_workspace_x_access_transparency: *indexSettings + so-logs-google_workspace_x_admin: *indexSettings + so-logs-google_workspace_x_alert: *indexSettings + so-logs-google_workspace_x_context_aware_access: *indexSettings + so-logs-google_workspace_x_device: *indexSettings + so-logs-google_workspace_x_drive: *indexSettings + so-logs-google_workspace_x_gcp: *indexSettings + so-logs-google_workspace_x_group_enterprise: *indexSettings + so-logs-google_workspace_x_groups: *indexSettings + so-logs-google_workspace_x_login: *indexSettings + so-logs-google_workspace_x_rules: *indexSettings + so-logs-google_workspace_x_saml: *indexSettings + so-logs-google_workspace_x_token: *indexSettings + so-logs-google_workspace_x_user_accounts: *indexSettings + so-logs-1password_x_item_usages: *indexSettings + so-logs-1password_x_signin_attempts: *indexSettings so-logs-osquery-manager-actions: *indexSettings - so-logs-osquery-manager-action.responses: *indexSettings - so-logs-elastic_agent.apm_server: *indexSettings - so-logs-elastic_agent.auditbeat: *indexSettings - so-logs-elastic_agent.cloudbeat: *indexSettings - so-logs-elastic_agent.endpoint_security: *indexSettings - so-logs-endpoint.alerts: *indexSettings - so-logs-endpoint.events.api: *indexSettings - so-logs-endpoint.events.file: *indexSettings - so-logs-endpoint.events.library: *indexSettings - so-logs-endpoint.events.network: *indexSettings - so-logs-endpoint.events.process: *indexSettings - so-logs-endpoint.events.registry: *indexSettings - so-logs-endpoint.events.security: *indexSettings - so-logs-elastic_agent.filebeat: *indexSettings - so-logs-elastic_agent.fleet_server: *indexSettings - so-logs-elastic_agent.heartbeat: *indexSettings + so-logs-osquery-manager-action_x_responses: *indexSettings + so-logs-elastic_agent_x_apm_server: *indexSettings + so-logs-elastic_agent_x_auditbeat: *indexSettings + so-logs-elastic_agent_x_cloudbeat: *indexSettings + so-logs-elastic_agent_x_endpoint_security: *indexSettings + so-logs-endpoint_x_alerts: *indexSettings + so-logs-endpoint_x_events_x_api: *indexSettings + so-logs-endpoint_x_events_x_file: *indexSettings + so-logs-endpoint_x_events_x_library: *indexSettings + so-logs-endpoint_x_events_x_network: *indexSettings + so-logs-endpoint_x_events_x_process: *indexSettings + so-logs-endpoint_x_events_x_registry: *indexSettings + so-logs-endpoint_x_events_x_security: *indexSettings + so-logs-elastic_agent_x_filebeat: *indexSettings + so-logs-elastic_agent_x_fleet_server: *indexSettings + so-logs-elastic_agent_x_heartbeat: *indexSettings so-logs-elastic_agent: *indexSettings - so-logs-elastic_agent.metricbeat: *indexSettings - so-logs-elastic_agent.osquerybeat: *indexSettings - so-logs-elastic_agent.packetbeat: *indexSettings + so-logs-elastic_agent_x_metricbeat: *indexSettings + so-logs-elastic_agent_x_osquerybeat: *indexSettings + so-logs-elastic_agent_x_packetbeat: *indexSettings so-case: *indexSettings so-common: *indexSettings so-endgame: *indexSettings diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 49d86d187..5fe0ed303 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -1,9 +1,11 @@ {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %} -{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %} -{% for index, settings in ES_INDEX_SETTINGS.items() %} +{%- set ES_INDEX_SETTINGS_ORIG = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %} +{% set ES_INDEX_SETTINGS = {} %} +{% for index, settings in ES_INDEX_SETTINGS_ORIG.items() %} {% if settings.index_template is defined %} {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} {% do settings.index_template.template.settings.index.pop('sort') %} {% endif %} {% endif %} + {% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_ORIG[index]}) %} {% endfor %} diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index afb8bdc67..b00fcbedf 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -6,8 +6,7 @@ . /usr/sbin/so-common -{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} -{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %} +{%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} {%- for index, settings in ES_INDEX_SETTINGS.items() %} {%- if settings.policy is defined %} From e43900074a3b2d02ec147ce69da786f9d0d6e9d6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 10 Aug 2023 11:54:49 -0400 Subject: [PATCH 41/56] ensure only 1 instance of so-rule-update runs. execute the cmd at the end of state run --- salt/idstools/enabled.sls | 6 ++-- salt/idstools/tools/sbin_jinja/so-rule-update | 34 +++++++++++-------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls index 966cb6786..3f5acda19 100644 --- a/salt/idstools/enabled.sls +++ b/salt/idstools/enabled.sls @@ -63,19 +63,21 @@ delete_so-idstools_so-status.disabled: so-rule-update: cron.present: - - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1 + - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download_cron.log 2>&1 - identifier: so-rule-update - user: root - minute: '1' - hour: '7' +# order this last to give so-idstools container time to be ready run_so-rule-update: cmd.run: - - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1' + - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download_idstools_state.log 2>&1' - require: - docker_container: so-idstools - onchanges: - file: idstoolsetcsync + - order: last {% else %} diff --git a/salt/idstools/tools/sbin_jinja/so-rule-update b/salt/idstools/tools/sbin_jinja/so-rule-update index 504831f9f..db110abc1 100755 --- a/salt/idstools/tools/sbin_jinja/so-rule-update +++ b/salt/idstools/tools/sbin_jinja/so-rule-update @@ -1,5 +1,9 @@ #!/bin/bash -. /usr/sbin/so-common + +# if this script isn't already running +if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then + + . /usr/sbin/so-common {%- from 'vars/globals.map.jinja' import GLOBALS %} {%- from 'idstools/map.jinja' import IDSTOOLSMERGED %} @@ -9,28 +13,30 @@ # Download the rules from the internet {%- if proxy %} -export http_proxy={{ proxy }} -export https_proxy={{ proxy }} -export no_proxy="{{ noproxy }}" + export http_proxy={{ proxy }} + export https_proxy={{ proxy }} + export no_proxy="{{ noproxy }}" {%- endif %} -mkdir -p /nsm/rules/suricata -chown -R socore:socore /nsm/rules/suricata + mkdir -p /nsm/rules/suricata + chown -R socore:socore /nsm/rules/suricata # Download the rules from the internet {%- if GLOBALS.airgap != 'True' %} {%- if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %} -docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force + docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force {%- elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %} -docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }} + docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }} {%- elif IDSTOOLSMERGED.config.ruleset == 'TALOS' %} -docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --url=https://www.snort.org/rules/snortrules-snapshot-2983.tar.gz?oinkcode={{ IDSTOOLSMERGED.config.oinkcode }} + docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --url=https://www.snort.org/rules/snortrules-snapshot-2983.tar.gz?oinkcode={{ IDSTOOLSMERGED.config.oinkcode }} {%- endif %} {%- endif %} -argstr="" -for arg in "$@"; do - argstr="${argstr} \"${arg}\"" -done + argstr="" + for arg in "$@"; do + argstr="${argstr} \"${arg}\"" + done -docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}" + docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}" + +fi From 4426437ad35e5b743bfb010edc8d511bc1f35270 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 10 Aug 2023 15:04:31 -0400 Subject: [PATCH 42/56] Update motd.md --- salt/soc/files/soc/motd.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/soc/files/soc/motd.md b/salt/soc/files/soc/motd.md index b5a4fac5f..cf22e863d 100644 --- a/salt/soc/files/soc/motd.md +++ b/salt/soc/files/soc/motd.md @@ -8,6 +8,10 @@ If you're ready to dive in, take a look at the [Alerts](/#/alerts) interface to To see all the latest features and fixes in this version of Security Onion, click the upper-right menu and then click the [What's New](/docs/release-notes.html) link. +## Enterprise Appliances + +Want the best hardware for your enterprise deployment? Check out our [enterprise appliances](https://securityonionsolutions.com/hardware/)! + ## Customize This Space Make this area your own by customizing the content in the [Config](/#/config?s=soc.files.soc.motd__md) interface. From caced64d1144d1f05797650a22caa979ebd2ac96 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 10 Aug 2023 16:10:39 -0400 Subject: [PATCH 43/56] set desktop background --- salt/desktop/files/00-background | 8 ++++++++ salt/desktop/xwindows.sls | 17 +++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 salt/desktop/files/00-background diff --git a/salt/desktop/files/00-background b/salt/desktop/files/00-background new file mode 100644 index 000000000..6f76c6408 --- /dev/null +++ b/salt/desktop/files/00-background @@ -0,0 +1,8 @@ +# Specify the dconf path +[org/gnome/desktop/background] + +# Specify the path to the desktop background image file +picture-uri='file:///usr/local/share/backgrounds/so-wallpaper.jpg' + +# Specify one of the rendering options for the background image: +picture-options='zoom' diff --git a/salt/desktop/xwindows.sls b/salt/desktop/xwindows.sls index c7790f9f4..b18109d45 100644 --- a/salt/desktop/xwindows.sls +++ b/salt/desktop/xwindows.sls @@ -35,6 +35,23 @@ convert_gnome_classic: {% endif %} {% endfor %} +desktop_wallpaper: + file.managed: + - name: /usr/local/share/backgrounds/so-wallpaper.jpg + - source: salt://desktop/files/so-wallpaper.jpg + - makedirs: True + +set_wallpaper: + file.managed: + - name: /etc/dconf/db/local.d/00-background + - source: salt://desktop/files/00-background + +run_dconf_update: + cmd.run: + - name: 'dconf update' + - onchanges: + - file: set_wallpaper + {% else %} desktop_xwindows_os_fail: From 0d894b7f527e8e35949aa01b2530294e0b1fe63e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 10 Aug 2023 18:57:17 -0400 Subject: [PATCH 44/56] Upgrade integration packages --- salt/elasticfleet/defaults.yaml | 7 +++++++ .../tools/sbin/so-elastic-fleet-common | 5 +++++ .../so-elastic-fleet-integration-policy-load | 3 +++ .../sbin_jinja/so-elastic-fleet-package-upgrade | 17 +++++++++++++++++ 4 files changed, 32 insertions(+) create mode 100644 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 0ae7a5176..c30d49dd4 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -29,8 +29,15 @@ elasticfleet: - azure - cloudflare - endpoint + - fleet_server - fim - github - google_workspace - log + - osquery_manager + - redis + - system + - tcp + - udp + - windows - 1password diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common index 73c36e5c8..197a111fb 100755 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common @@ -56,6 +56,11 @@ elastic_fleet_package_version_check() { curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.version' } +elastic_fleet_package_latest_version_check() { + PACKAGE=$1 + curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.latestVersion' +} + elastic_fleet_package_install() { PKGKEY=$1 curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PKGKEY" diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index 501aafbda..ae0fbb6ba 100755 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -9,6 +9,9 @@ RETURN_CODE=0 if [ ! -f /opt/so/state/eaintegrations.txt ]; then + # First, check for any package upgrades + /usr/sbin/so-elastic-fleet-package-upgrade + # Initial Endpoints for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json do diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade new file mode 100644 index 000000000..81eb01534 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. +{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} +{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} + +. /usr/sbin/so-elastic-fleet-common + +{%- for PACKAGE in SUPPORTED_PACKAGES %} +echo "Upgrading {{ PACKAGE }} package..." +VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}") +elastic_fleet_package_install "{{ PACKAGE }}-$VERSION" +echo +{%- endfor %} +echo From 1d83b2f2e644aa8a68b7982265f91ce12c98e60e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 10 Aug 2023 19:51:12 -0400 Subject: [PATCH 45/56] Add elasticsearch integration --- salt/elasticfleet/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index c30d49dd4..cb282aade 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -28,6 +28,7 @@ elasticfleet: - aws - azure - cloudflare + - elasticsearch - endpoint - fleet_server - fim From fdb2ca4167311f9044299a548337ecfcb4943351 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 11 Aug 2023 09:15:41 -0400 Subject: [PATCH 46/56] set SO desktop wallpaper for iso install --- setup/so-functions | 9 +++++++++ setup/so-setup | 2 ++ 2 files changed, 11 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index d138d97df..1a1eb1919 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2302,6 +2302,15 @@ set_default_log_size() { log_size_limit=$( echo "$disk_size_gb" "$percentage" | awk '{printf("%.0f", $1 * ($2/100))}') } +set_desktop_background() { + + logCmd "mkdir /usr/local/share/backgrounds" + logCmd "cp ../salt/desktop/files/so-wallpaper.jpg /usr/local/share/backgrounds/so-wallpaper.jpg" + logCmd "cp ../salt/desktop/files/00-background /etc/dconf/db/local.d/00-background" + logCmd "dconf update" + +} + set_hostname() { logCmd "hostnamectl set-hostname --static $HOSTNAME" diff --git a/setup/so-setup b/setup/so-setup index ccc9f6f2f..d048cc8bc 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -341,6 +341,8 @@ if [[ $is_desktop ]]; then securityonion_repo info "Enabling graphical interface and setting it to load at boot" systemctl set-default graphical.target + info "Setting desktop background" + set_desktop_background echo "Desktop Install Complete!" echo "" echo "Please reboot to start graphical interface." From 3f054031a0bc1e3ac5932d1c5437919a698dcf81 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 11 Aug 2023 13:32:22 -0400 Subject: [PATCH 47/56] Set default for import and eval only --- .../tools/sbin_jinja/so-elastic-fleet-es-url-update | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update index 3b0e01f43..481287eef 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update @@ -12,9 +12,13 @@ if ! is_manager_node; then fi function update_es_urls() { - # Generate updated JSON payload + + # Generate updated JSON payload +{% if grains.role not in ['so-import', 'so-eval'] %} JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"config_yaml":""}') - +{%- else %} + JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}') +{%- endif %} # Update Fleet Elasticsearch URLs curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" } From a5e60363cf41793de89026c6f55d40ab2ad8c7d7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 11 Aug 2023 13:38:16 -0400 Subject: [PATCH 48/56] add missing annotations to avoid soc crash --- salt/soc/soc_soc.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index e3d704e80..03fd47e80 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -45,9 +45,10 @@ soc: actions: description: A list of actions a user can take from the SOC UI against a hunt, alert, and other records. The action must be defined in JSON object format, and contain a "name" key and "links" key. The links is a list of URLs, where the most suitable URL in the list will be the selected URL when the user clicks the action. global: True + forcedType: "[]{}" eventFields: default: - description: The list of fields to show as columns in the Hunt/Dashboards event table, when no other specific mapping applies. Mappings are defined by the format ":event.module:event.dataset". + description: Event fields mappings are defined by the format ":event.module:event.dataset", so if you would like to customize which fields show for syslog events of originating from zeek you will find that entry in the left panel that looks like :zeek:syslog. This default entry is used for all events that do not match an existing mapping defined on the left side of this configuration screen. global: True advanced: True server: @@ -139,6 +140,7 @@ soc: description: List of available external tools visible in the SOC UI. Each tool is defined in JSON object notation, and must include the "name" key and "link" key, where the link is the tool's URL. global: True advanced: True + forcedType: "[]{}" hunt: &appSettings groupItemsPerPage: description: Default number of aggregations to show per page. Larger values consume more vertical area in the SOC UI. @@ -164,6 +166,12 @@ soc: queries: description: List of default queries to show in the query list. Each query is represented in JSON object notation, and must include the "name" key and "query" key. global: True + forcedType: "[]{}" + queryToggleFilters: + description: Customize togglable query filters that apply to all queries. Exclusive toggles will invert the filter if toggled off rather than omitting the filter from the query. + global: True + advanced: True + forcedType: "[]{}" alerts: *appSettings cases: *appSettings dashboards: *appSettings From 1fb3a595735fdbebf9eed6e8649b5a919d1f2f61 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 11 Aug 2023 13:41:58 -0400 Subject: [PATCH 49/56] add missing annotations to avoid soc crash --- salt/soc/soc_soc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 03fd47e80..b2ed893f6 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -48,7 +48,7 @@ soc: forcedType: "[]{}" eventFields: default: - description: Event fields mappings are defined by the format ":event.module:event.dataset", so if you would like to customize which fields show for syslog events of originating from zeek you will find that entry in the left panel that looks like :zeek:syslog. This default entry is used for all events that do not match an existing mapping defined on the left side of this configuration screen. + description: Event fields mappings are defined by the format ":event.module:event.dataset". For example, to customize which fields show for 'syslog' events originating from 'zeek', find the eventField item in the left panel that looks like ':zeek:syslog'. This 'default' entry is used for all events that do not match an existing mapping defined in the list to the left. global: True advanced: True server: From ce63e47fcdc5b134e9b94c048d35e0d86ee9eae4 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 11 Aug 2023 14:47:33 -0400 Subject: [PATCH 50/56] Enable forced update --- .../tools/sbin_jinja/so-elastic-fleet-es-url-update | 7 +++++++ salt/manager/tools/sbin/soup | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update index 481287eef..5d5b7e7e0 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update @@ -46,6 +46,13 @@ NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "$ NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') # Compare the current & new list of URLs - if different, update the Fleet Elasticsearch URLs +if [ "$1" = "--force" ]; then + printf "\nUpdating List, since --force was specified.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + update_es_urls + exit 0 +fi + if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then printf "\nHashes match - no update needed.\n" printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index ebc9c8074..a41d60648 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -425,7 +425,8 @@ post_to_2.4.5() { } post_to_2.4.10() { - echo "Nothing to apply" + echo "Updating Elastic Fleet ES URLs...." + /sbin/so-elastic-fleet-es-url-update --force POSTVERSION=2.4.10 } From 00297cd864706036628bf607e580da1bc5164f07 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 11 Aug 2023 16:10:16 -0400 Subject: [PATCH 51/56] Move from post to pre --- salt/manager/tools/sbin/soup | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index a41d60648..8054fb14d 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -425,8 +425,7 @@ post_to_2.4.5() { } post_to_2.4.10() { - echo "Updating Elastic Fleet ES URLs...." - /sbin/so-elastic-fleet-es-url-update --force + echo "Nothing to apply" POSTVERSION=2.4.10 } @@ -491,8 +490,8 @@ up_to_2.4.5() { } up_to_2.4.10() { - echo "Nothing to do for 2.4.10" - + echo "Updating Elastic Fleet ES URLs...." + /sbin/so-elastic-fleet-es-url-update --force INSTALLEDVERSION=2.4.10 } From f38b77892b4a5c94314694e11f7f1523728232c1 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 11 Aug 2023 17:14:48 -0400 Subject: [PATCH 52/56] Move back --- salt/manager/tools/sbin/soup | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 8054fb14d..b242fd279 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -425,7 +425,8 @@ post_to_2.4.5() { } post_to_2.4.10() { - echo "Nothing to apply" + echo "Updating Elastic Fleet ES URLs...." + /sbin/so-elastic-fleet-es-url-update --force POSTVERSION=2.4.10 } @@ -490,8 +491,8 @@ up_to_2.4.5() { } up_to_2.4.10() { - echo "Updating Elastic Fleet ES URLs...." - /sbin/so-elastic-fleet-es-url-update --force + echo "Nothing to do for 2.4.10" + INSTALLEDVERSION=2.4.10 } From f3a58cd336175fb7c8b4781315a330e26057c0ee Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Sun, 13 Aug 2023 16:46:32 -0400 Subject: [PATCH 53/56] soup should respect current indentation in soc_global.sls --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b242fd279..af09cc9df 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -562,7 +562,7 @@ update_version() { echo "Updating the Security Onion version file." echo $NEWVERSION > /etc/soversion echo $HOTFIXVERSION > /etc/sohotfix - sed -i "/ soversion:/c\ soversion: $NEWVERSION" /opt/so/saltstack/local/pillar/global/soc_global.sls + sed -i "s/soversion:.*/soversion: $NEWVERSION/" /opt/so/saltstack/local/pillar/global/soc_global.sls } upgrade_check() { From 4106d1f69d624467b1b123a81a1a7ee3784636ea Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 14 Aug 2023 16:33:08 -0400 Subject: [PATCH 54/56] 2.4.10 --- DOWNLOAD_AND_VERIFY_ISO.md | 16 ++++++++-------- sigs/securityonion-2.4.10-20230815.iso.sig | Bin 0 -> 566 bytes 2 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 sigs/securityonion-2.4.10-20230815.iso.sig diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index b9b3da297..7fed2991c 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,18 +1,18 @@ -### 2.4.5-20230807 ISO image released on 2023/08/07 +### 2.4.10-20230815 ISO image released on 2023/08/07 ### Download and Verify -2.4.5-20230807 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.4.5-20230807.iso +2.4.10-20230815 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.4.10-20230815.iso MD5: F83FD635025A3A65B380EAFCEB61A92E SHA1: 5864D4CD520617E3328A3D956CAFCC378A8D2D08 SHA256: D333BAE0DD198DFD80DF59375456D228A4E18A24EDCDB15852CD4CA3F92B69A7 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.5-20230807.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.10-20230815.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS @@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2. Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.5-20230807.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.10-20230815.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.4.5-20230807.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.4.10-20230815.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.4.5-20230807.iso.sig securityonion-2.4.5-20230807.iso +gpg --verify securityonion-2.4.10-20230815.iso.sig securityonion-2.4.10-20230815.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Sat 05 Aug 2023 10:12:46 AM EDT using RSA key ID FE507013 +gpg: Signature made Sun 13 Aug 2023 05:30:29 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.4.10-20230815.iso.sig b/sigs/securityonion-2.4.10-20230815.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..636dfe63b7dcce9057c6f73a795234f32821fd18 GIT binary patch literal 566 zcmV-60?GY}0y6{v0SEvc79j-41gSkXz6^6dp_W8^5Ma0dP;e6k0%X}sbpQ$p5PT3| zxBgIY6Xz`t|9z8nQ?NY-QP9ZVJcc{@E~xDra#)inhc&W^+E5YB0bD^Bzk#8o&v9UB zxd(n%%3G*3*hRPWTldLgI9~ntBKzxDYiPlkWr~(0Jx4XeyrA~uYHcayLQyh<`6U0? z`#=nl;KmqW(T5QYQHAMp+t~zk{yAcJ8M9l2 zf2L=<>F}kZ!^Q1QANb2fn?Rd(9z}#a+tD;zwxczvdTxD@$YDHnX14+3t z=m|8_td=SDU~~}Ve&|n>{W(>gg&;1M%$>SZ5#_K#;IF{ONymNV_^X`^sxq#_CfwN5 zyh33;LJA;Y%VYqwn~=mVL(OmODVn|zJPyvXXza5!Z EXqbu^eE Date: Mon, 14 Aug 2023 16:34:32 -0400 Subject: [PATCH 55/56] 2.4.10 --- DOWNLOAD_AND_VERIFY_ISO.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 7fed2991c..980bb062f 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -7,9 +7,9 @@ 2.4.10-20230815 ISO image: https://download.securityonion.net/file/securityonion/securityonion-2.4.10-20230815.iso -MD5: F83FD635025A3A65B380EAFCEB61A92E -SHA1: 5864D4CD520617E3328A3D956CAFCC378A8D2D08 -SHA256: D333BAE0DD198DFD80DF59375456D228A4E18A24EDCDB15852CD4CA3F92B69A7 +MD5: 97AEC929FB1FC22F106C0C93E3476FAB +SHA1: 78AF37FD19FDC34BA324C1A661632D19D1F2284A +SHA256: D04BA45D1664FC3CF7EA2188CB7E570642F6390C3959B4AFBB8222A853859394 Signature for ISO image: https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.10-20230815.iso.sig From ad9da07de1ef4ca7e307cec31dab3d615064d341 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 14 Aug 2023 16:51:24 -0400 Subject: [PATCH 56/56] Update DOWNLOAD_AND_VERIFY_ISO.md --- DOWNLOAD_AND_VERIFY_ISO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 980bb062f..816c4f827 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,4 +1,4 @@ -### 2.4.10-20230815 ISO image released on 2023/08/07 +### 2.4.10-20230815 ISO image released on 2023/08/15