From 1d2534b2a127f4cfc605e4d7f68ad71af56bcdf8 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 29 Jul 2022 08:24:57 -0400 Subject: [PATCH 01/31] Increment version --- HOTFIX | 2 +- VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HOTFIX b/HOTFIX index 8ab213017..d3f5a12fa 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +1 @@ -20220719 + diff --git a/VERSION b/VERSION index 3994a975c..70a2b29d7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.140 +2.3.150 From 2cc665bac635d961f9f1fe20570fd0d77dcee921 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 29 Jul 2022 09:55:20 -0400 Subject: [PATCH 02/31] https://github.com/Security-Onion-Solutions/securityonion/issues/8404 --- salt/top.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/top.sls b/salt/top.sls index 87f96143f..27193f7ac 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -84,7 +84,9 @@ base: {%- if STRELKA %} - strelka {%- endif %} + {%- if FILEBEAT %} - filebeat + {%- endif %} {%- if FLEETMANAGER or FLEETNODE %} - fleet.install_package {%- endif %} @@ -433,7 +435,9 @@ base: - redis - fleet - fleet.install_package + {%- if FILEBEAT %} - filebeat + {%- endif %} - schedule - docker_clean @@ -507,7 +511,9 @@ base: {%- endif %} - schedule - docker_clean + {%- if FILEBEAT %} - filebeat + {%- endif %} - idh 'J@workstation:gui:enabled:^[Tt][Rr][Uu][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:CentOS )': From 4c1585f8d8fecfad2a3000eabb94378339e6b353 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 29 Jul 2022 14:50:10 -0400 Subject: [PATCH 03/31] FIX: Display PCAP menu action on Dashboards page #8343 --- salt/soc/files/soc/menu.actions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/files/soc/menu.actions.json b/salt/soc/files/soc/menu.actions.json index 8af63f2a8..0a9894c89 100644 --- a/salt/soc/files/soc/menu.actions.json +++ b/salt/soc/files/soc/menu.actions.json @@ -19,7 +19,7 @@ "/joblookup?esid={:soc_id}&time={:@timestamp}", "/joblookup?ncid={:network.community_id}&time={:@timestamp}" ], - "categories": ["hunt", "alerts"]}, + "categories": ["hunt", "alerts", "dashboards"]}, { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", "links": [ "/cyberchef/#input={value|base64}" From 10ba3b4b5a771193d8694fabe005d85e623e8236 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 29 Jul 2022 16:30:12 -0400 Subject: [PATCH 04/31] increment version --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d5a8586cf..170bb0039 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -## Security Onion 2.3.140 +## Security Onion 2.3.150 -Security Onion 2.3.140 is here! +Security Onion 2.3.150 is here! ## Screenshots From d24125c9e61fa4ba5fb4c0b6f173fb12ab65ece9 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 1 Aug 2022 10:40:57 -0400 Subject: [PATCH 05/31] Update Elastic version to 8.3.3 --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index 4ec8f9ca7..a8dc56f32 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.3.2","id": "8.3.2","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.3.3","id": "8.3.3","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From 3123407ef0c34ceb977c9d803909f53fde5287e6 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 1 Aug 2022 10:41:39 -0400 Subject: [PATCH 06/31] Update Elastic version to 8.3.3 --- salt/kibana/bin/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index e19f25439..7b49f5a94 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -59,7 +59,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/8.3.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/8.3.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From 839cfcaefa3e93ae0d20fb8aad7fca70813fcab2 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 2 Aug 2022 14:32:17 +0000 Subject: [PATCH 07/31] Update Elasticsearch defaults file and config.map.jinja to allow for local GeoIP database use when airgap is enabled --- salt/elasticsearch/config.map.jinja | 5 +++++ salt/elasticsearch/defaults.yaml | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 9a80ce30f..71ff5b912 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,5 +1,6 @@ {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} +{% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %} {% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} {% do ESCONFIG.elasticsearch.config.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} @@ -33,6 +34,10 @@ {% endif %} {% endif %} +{% if ISAIRGAP %} + {% do ESCONFIG.elasticsearch.config.ingest.geoip.downloader.update({'enabled': false}) %} +{% endif %} + {# merge with the elasticsearch pillar #} {% set ESCONFIG = salt['pillar.get']('elasticsearch:config', default=ESCONFIG.elasticsearch.config, merge=True) %} diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 830d1372c..782f2ad93 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -55,6 +55,10 @@ elasticsearch: indices: id_field_data: enabled: false + ingest: + geoip: + downloader: + enabled: true logger: org: elasticsearch: From c69cac0e5f7e66ed75ca93477ca2d748b6da2240 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 2 Aug 2022 11:31:35 -0400 Subject: [PATCH 08/31] Update Kibana version to 8.3.3 --- salt/kibana/bin/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index e19f25439..7b49f5a94 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -59,7 +59,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/8.3.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/8.3.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From f2b10a5a862b62a45449cd904c4499ad6aa27276 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 2 Aug 2022 11:32:01 -0400 Subject: [PATCH 09/31] Update Kibana version to 8.3.3 --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index 4ec8f9ca7..a8dc56f32 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.3.2","id": "8.3.2","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.3.3","id": "8.3.3","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From db8d9fff2c2f64c0fdf5ddeb5eb2172582aded36 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 2 Aug 2022 16:22:26 -0400 Subject: [PATCH 10/31] manage salt-minion start delay with systemd drop-in file - https://github.com/Security-Onion-Solutions/securityonion/issues/8441 --- salt/salt/map.jinja | 2 -- salt/salt/minion.sls | 6 +++--- salt/salt/service/salt-minion.service.jinja | 15 --------------- salt/salt/service/start-delay.conf.jinja | 2 ++ 4 files changed, 5 insertions(+), 20 deletions(-) delete mode 100644 salt/salt/service/salt-minion.service.jinja create mode 100644 salt/salt/service/start-delay.conf.jinja diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 389a95607..eb9f5ae89 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -11,7 +11,6 @@ {% set PYTHON3INFLUX= 'influxdb == ' ~ PYTHONINFLUXVERSION %} {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'python-dateutil', 'pytz', 'requests'] %} {% set PYTHONINSTALLER = 'pip' %} - {% set SYSTEMD_UNIT_FILE = '/lib/systemd/system/salt-minion.service' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %} @@ -22,7 +21,6 @@ {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} {% set PYTHONINSTALLER = 'pkg' %} - {% set SYSTEMD_UNIT_FILE = '/usr/lib/systemd/system/salt-minion.service' %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 15e203d82..fafb6f0f3 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -81,10 +81,10 @@ set_log_levels: - "log_level: error" - "log_level_logfile: error" -salt_minion_service_unit_file: +salt_minion_service_start_delay: file.managed: - - name: {{ SYSTEMD_UNIT_FILE }} - - source: salt://salt/service/salt-minion.service.jinja + - name: /etc/systemd/system/salt-minion.service.d/start-delay.conf + - source: salt://salt/service/start-delay.conf.jinja - template: jinja - defaults: service_start_delay: {{ service_start_delay }} diff --git a/salt/salt/service/salt-minion.service.jinja b/salt/salt/service/salt-minion.service.jinja deleted file mode 100644 index c7bae0bc2..000000000 --- a/salt/salt/service/salt-minion.service.jinja +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=The Salt Minion -Documentation=man:salt-minion(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltstack.com/en/latest/contents.html -After=network.target salt-master.service - -[Service] -KillMode=process -Type=notify -NotifyAccess=all -LimitNOFILE=8192 -ExecStart=/usr/bin/salt-minion -ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/salt/salt/service/start-delay.conf.jinja b/salt/salt/service/start-delay.conf.jinja new file mode 100644 index 000000000..33917b174 --- /dev/null +++ b/salt/salt/service/start-delay.conf.jinja @@ -0,0 +1,2 @@ +[Service] +ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }} From 9ac640fa67efc3f9dad307846111b517b24b23a2 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 3 Aug 2022 09:21:03 -0400 Subject: [PATCH 11/31] Remove airgap-specific logic for ingest.geoip.downloader --- salt/elasticsearch/config.map.jinja | 5 ----- 1 file changed, 5 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 71ff5b912..9a80ce30f 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,6 +1,5 @@ {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} -{% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %} {% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} {% do ESCONFIG.elasticsearch.config.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} @@ -34,10 +33,6 @@ {% endif %} {% endif %} -{% if ISAIRGAP %} - {% do ESCONFIG.elasticsearch.config.ingest.geoip.downloader.update({'enabled': false}) %} -{% endif %} - {# merge with the elasticsearch pillar #} {% set ESCONFIG = salt['pillar.get']('elasticsearch:config', default=ESCONFIG.elasticsearch.config, merge=True) %} From 8c694a7ca3f4b09f25f06e5d6eb2f2d3722bbf55 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 3 Aug 2022 09:21:40 -0400 Subject: [PATCH 12/31] Disable ingest.geoip.downloader by default --- salt/elasticsearch/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 782f2ad93..ee2051d12 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -58,7 +58,7 @@ elasticsearch: ingest: geoip: downloader: - enabled: true + enabled: false logger: org: elasticsearch: From 15f7fd892074d07827c77d085efc21f17d08f555 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 3 Aug 2022 15:16:12 -0400 Subject: [PATCH 13/31] fix the requisite --- salt/salt/minion.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index fafb6f0f3..fc6546392 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -109,7 +109,7 @@ salt_minion_service: - file: mine_functions {% if INSTALLEDSALTVERSION|string == SALTVERSION|string %} - file: set_log_levels - - file: salt_minion_service_unit_file + - file: salt_minion_service_start_delay {% endif %} - order: last From 99805cc326695d61196d76a6d99cd55d64a290f0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 3 Aug 2022 16:54:22 -0400 Subject: [PATCH 14/31] ensure parent dirs are created --- salt/salt/minion.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index fc6546392..7af96ff07 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -86,6 +86,7 @@ salt_minion_service_start_delay: - name: /etc/systemd/system/salt-minion.service.d/start-delay.conf - source: salt://salt/service/start-delay.conf.jinja - template: jinja + - makedirs: True - defaults: service_start_delay: {{ service_start_delay }} - onchanges_in: From f02f431dab2e7d6a712e8944084f4148d79783d5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 4 Aug 2022 16:34:06 -0400 Subject: [PATCH 15/31] ensure ExecStartPre is removed from default salt-minion service file --- salt/salt/minion.sls | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 7af96ff07..b3110a3f9 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -81,6 +81,14 @@ set_log_levels: - "log_level: error" - "log_level_logfile: error" +delete_pre_150_start_delay: + file.line: + - name: {{ SYSTEMD_UNIT_FILE }} + - match: ^ExecStartPre=* + - mode: delete + - onchanges_in: + - module: systemd_reload + salt_minion_service_start_delay: file.managed: - name: /etc/systemd/system/salt-minion.service.d/start-delay.conf From 3b62fc63c9dde9ad7c005046b0b6b580f31ea3e6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 5 Aug 2022 10:53:07 -0400 Subject: [PATCH 16/31] add SYSTEMD_UNIT_FILE back to map file --- salt/salt/map.jinja | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index eb9f5ae89..389a95607 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -11,6 +11,7 @@ {% set PYTHON3INFLUX= 'influxdb == ' ~ PYTHONINFLUXVERSION %} {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'python-dateutil', 'pytz', 'requests'] %} {% set PYTHONINSTALLER = 'pip' %} + {% set SYSTEMD_UNIT_FILE = '/lib/systemd/system/salt-minion.service' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %} @@ -21,6 +22,7 @@ {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} {% set PYTHONINSTALLER = 'pkg' %} + {% set SYSTEMD_UNIT_FILE = '/usr/lib/systemd/system/salt-minion.service' %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} From cb384ae0246f1472f805ee865bf772529f670aba Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 5 Aug 2022 11:25:33 -0400 Subject: [PATCH 17/31] Ensure check_local_mods() runs at the beginning of SOUP, in addition to the end, and also that it prompts (forces) the user to accept/review local modifications. --- salt/common/tools/sbin/soup | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d41fcdfcf..d42cb59fa 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -203,7 +203,7 @@ check_airgap() { check_local_mods() { local salt_local=/opt/so/saltstack/local - + local_ignore_arr=("/opt/so/saltstack/local/salt/zeek/policy/intel/intel.dat") local_mod_arr=() while IFS= read -r -d '' local_file; do @@ -211,8 +211,10 @@ check_local_mods() { default_file="${DEFAULT_SALT_DIR}${stripped_path}" if [[ -f $default_file ]]; then file_diff=$(diff "$default_file" "$local_file" ) - if [[ $(echo "$file_diff" | grep -c "^<") -gt 0 ]]; then - local_mod_arr+=( "$local_file" ) + if [[ ! " ${local_ignore_arr[*]} " =~ " ${local_file} " ]]; then + if [[ $(echo "$file_diff" | grep -c "^<") -gt 0 ]]; then + local_mod_arr+=( "$local_file" ) + fi fi fi done< <(find $salt_local -type f -print0) @@ -223,11 +225,24 @@ check_local_mods() { echo " $file_str" done echo "" - echo "To reference this list later, check $SOUP_LOG" - sleep 10 + echo "To reference this list later, check $SOUP_LOG". + echo + if [[ -z $UNATTENDED ]]; then + while true; do + read -p "Please review the local modifications shown above as they may cause problems during or after the update. + +Would you like to proceed with the update anyway? + +If so, type 'YES'. Otherwise, type anything else to exit SOUP." yn + + case $yn in + [yY][eE][sS] ) echo "Local modifications accepted. Continuing..."; break;; + * ) exit 0;; + esac + done + fi fi } - # {% endraw %} check_pillar_items() { @@ -1230,6 +1245,7 @@ main() { echo "Let's see if we need to update Security Onion." upgrade_check + check_local_mods upgrade_space echo "Checking for Salt Master and Minion updates." From d7906945df2256a688607afac3f9937c9f911fb4 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 Aug 2022 08:24:38 -0400 Subject: [PATCH 18/31] Add extra set of brackets for comparison of integers --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d42cb59fa..5f176a264 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -463,7 +463,7 @@ enable_highstate() { es_version_check() { CHECK_ES=$(echo $INSTALLEDVERSION | awk -F. '{print $3}') - if [ "$CHECK_ES" -lt "110" ]; then + if [[ "$CHECK_ES" -lt "110" ]]; then echo "You are currently running Security Onion $INSTALLEDVERSION. You will need to update to version 2.3.130 before updating to 2.3.140 or higher." echo "" echo "If your deployment has Internet access, you can use the following command to update to 2.3.130:" From fd7a118664a7526a15e101876b92533f9ca0754c Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 Aug 2022 08:58:19 -0400 Subject: [PATCH 19/31] Invoke check_local_mods() function earlier so we don't have to wait for Docker image downloads or OS updates before checking and potentially exiting SOUP --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 5f176a264..09d1dc141 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -1233,6 +1233,7 @@ main() { set_palette check_elastic_license echo "" + check_local_mods check_os_updates echo "Generating new repo archive" @@ -1245,7 +1246,6 @@ main() { echo "Let's see if we need to update Security Onion." upgrade_check - check_local_mods upgrade_space echo "Checking for Salt Master and Minion updates." From 4c677961c4bbf35b7d0ee3cc3a14b4fb5a857b24 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 11 Aug 2022 08:49:25 -0400 Subject: [PATCH 20/31] FIX: Fix TLP options in Cases to align with TLP 2.0 #8469 --- salt/soc/files/soc/presets.tlp.json | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/soc/files/soc/presets.tlp.json b/salt/soc/files/soc/presets.tlp.json index 6ef37164d..5ae860b28 100644 --- a/salt/soc/files/soc/presets.tlp.json +++ b/salt/soc/files/soc/presets.tlp.json @@ -1,9 +1,10 @@ { "labels": [ - "white", - "green", - "amber", - "red" + "CLEAR", + "GREEN", + "AMBER", + "AMBER+STRICT", + "RED" ], "customEnabled": false -} \ No newline at end of file +} From 4003876465bb26abafc2ce0ec5036f24fb88d849 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 11 Aug 2022 08:49:54 -0400 Subject: [PATCH 21/31] FIX: Fix TLP options in Cases to align with TLP 2.0 #8469 --- salt/soc/files/soc/presets.pap.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/soc/files/soc/presets.pap.json b/salt/soc/files/soc/presets.pap.json index 6ef37164d..22aca7536 100644 --- a/salt/soc/files/soc/presets.pap.json +++ b/salt/soc/files/soc/presets.pap.json @@ -1,9 +1,9 @@ { "labels": [ - "white", - "green", - "amber", - "red" + "WHITE", + "GREEN", + "AMBER", + "RED" ], "customEnabled": false -} \ No newline at end of file +} From 7bf26034140f78f206b2b32439e7f4dbb6e0733b Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 11 Aug 2022 15:32:49 -0400 Subject: [PATCH 22/31] revert to lower case #8469 --- salt/soc/files/soc/presets.pap.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/soc/files/soc/presets.pap.json b/salt/soc/files/soc/presets.pap.json index 22aca7536..8b254b020 100644 --- a/salt/soc/files/soc/presets.pap.json +++ b/salt/soc/files/soc/presets.pap.json @@ -1,9 +1,9 @@ { "labels": [ - "WHITE", - "GREEN", - "AMBER", - "RED" + "white", + "green", + "amber", + "red" ], "customEnabled": false } From 32c29b28eba30bf6769539bf449b88ebedf948a6 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 11 Aug 2022 15:33:30 -0400 Subject: [PATCH 23/31] revert to lower case #8469 --- salt/soc/files/soc/presets.tlp.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/soc/files/soc/presets.tlp.json b/salt/soc/files/soc/presets.tlp.json index 5ae860b28..5cefe4ada 100644 --- a/salt/soc/files/soc/presets.tlp.json +++ b/salt/soc/files/soc/presets.tlp.json @@ -1,10 +1,10 @@ { "labels": [ - "CLEAR", - "GREEN", - "AMBER", - "AMBER+STRICT", - "RED" + "clear", + "green", + "amber", + "amber+strict", + "red" ], "customEnabled": false } From 179f669acfbaebe719f96b3bba83d9edca394054 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 12 Aug 2022 13:10:47 -0400 Subject: [PATCH 24/31] FIX: so-curator-closed-delete-delete needs to reference new Elasticsearch directory #8529 --- salt/curator/files/bin/so-curator-closed-delete-delete | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/files/bin/so-curator-closed-delete-delete b/salt/curator/files/bin/so-curator-closed-delete-delete index b872a7aeb..5476b1390 100755 --- a/salt/curator/files/bin/so-curator-closed-delete-delete +++ b/salt/curator/files/bin/so-curator-closed-delete-delete @@ -29,7 +29,7 @@ LOG="/opt/so/log/curator/so-curator-closed-delete.log" overlimit() { - [[ $(du -hs --block-size=1GB /nsm/elasticsearch/nodes | awk '{print $1}' ) -gt "{{LOG_SIZE_LIMIT}}" ]] + [[ $(du -hs --block-size=1GB /nsm/elasticsearch/indices | awk '{print $1}' ) -gt "{{LOG_SIZE_LIMIT}}" ]] } closedindices() { From 2dfd41bd3c64913bd9c17d8e41007d8e5353df32 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 17 Aug 2022 09:17:27 -0400 Subject: [PATCH 25/31] remove pipeline time panel - https://github.com/Security-Onion-Solutions/securityonion/issues/8369 --- salt/grafana/defaults.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/salt/grafana/defaults.yaml b/salt/grafana/defaults.yaml index 024fd5dfd..8714381d3 100644 --- a/salt/grafana/defaults.yaml +++ b/salt/grafana/defaults.yaml @@ -3085,12 +3085,6 @@ grafana: y: 16 h: 8 w: 24 - elasticsearch_pipeline_time_nontc_graph: - gridPos: - x: 0 - y: 24 - h: 8 - w: 24 pipeline_overview_tc: @@ -3140,9 +3134,3 @@ grafana: y: 16 h: 8 w: 24 - elasticsearch_pipeline_time_tc_graph: - gridPos: - x: 0 - y: 24 - h: 8 - w: 24 From 5deda45b6618bd722f78b36cb5ce05fb139701a7 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 18 Aug 2022 09:11:38 -0400 Subject: [PATCH 26/31] Update elastalert_indices_check() function to only delete Elastalert indices if major Elasticsearch version is less than 8 Update elastalert_indices_check() function to only delete Elastalert indices if major Elasticsearch version is less than 8. Also clean up the output to only emit one notification regarding index deletion, and additional verbiage around function operation. --- salt/common/tools/sbin/soup | 85 ++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 39 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 09d1dc141..85ef432d1 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -387,12 +387,7 @@ clone_to_tmp() { } elastalert_indices_check() { - - # Stop Elastalert to prevent Elastalert indices from being re-created - if grep -q "^so-elastalert$" /opt/so/conf/so-status/so-status.conf ; then - so-elastalert-stop || true - fi - + echo "Checking Elastalert indices for compatibility..." # Wait for ElasticSearch to initialize echo -n "Waiting for ElasticSearch..." COUNT=0 @@ -409,8 +404,8 @@ elastalert_indices_check() { echo -n "." fi done - - # Unable to connect to Elasticsearch + + # Unable to connect to Elasticsearch if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then echo echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" @@ -418,39 +413,51 @@ elastalert_indices_check() { exit 1 fi - # Check Elastalert indices - echo "Deleting Elastalert indices to prevent issues with upgrade to Elastic 8..." - CHECK_COUNT=0 - while [[ "$CHECK_COUNT" -le 2 ]]; do - # Delete Elastalert indices - for i in $(so-elasticsearch-query _cat/indices | grep elastalert | awk '{print $3}'); do - so-elasticsearch-query $i -XDELETE; + MAJOR_ES_VERSION=$(so-elasticsearch-query / | jq -r .version.number | cut -d '.' -f1) + if [[ "$MAJOR_ES_VERSION" -lt "8" ]]; then + + # Stop Elastalert to prevent Elastalert indices from being re-created + if grep -q "^so-elastalert$" /opt/so/conf/so-status/so-status.conf ; then + so-elastalert-stop || true + fi + + # Check Elastalert indices + echo "Deleting Elastalert indices to prevent issues with upgrade to Elastic 8..." + CHECK_COUNT=0 + while [[ "$CHECK_COUNT" -le 2 ]]; do + # Delete Elastalert indices + for i in $(so-elasticsearch-query _cat/indices | grep elastalert | awk '{print $3}'); do + so-elasticsearch-query $i -XDELETE; + done + + # Check to ensure Elastalert indices are deleted + COUNT=0 + ELASTALERT_INDICES_DELETED="no" + while [[ "$COUNT" -le 240 ]]; do + RESPONSE=$(so-elasticsearch-query "elastalert*") + if [[ "$RESPONSE" == "{}" ]]; then + ELASTALERT_INDICES_DELETED="yes" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi + done + ((CHECK_COUNT+=1)) done - # Check to ensure Elastalert indices are deleted - COUNT=0 - ELASTALERT_INDICES_DELETED="no" - while [[ "$COUNT" -le 240 ]]; do - RESPONSE=$(so-elasticsearch-query elastalert*) - if [[ "$RESPONSE" == "{}" ]]; then - ELASTALERT_INDICES_DELETED="yes" - echo "Elastalert indices successfully deleted." - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." - fi - done - ((CHECK_COUNT+=1)) - done - - # If we were unable to delete the Elastalert indices, exit the script - if [ "$ELASTALERT_INDICES_DELETED" == "no" ]; then - echo - echo -e "Unable to connect to delete Elastalert indices. Exiting." - echo - exit 1 + # If we were unable to delete the Elastalert indices, exit the script + if [ "$ELASTALERT_INDICES_DELETED" == "yes" ]; then + echo "Elastalert indices successfully deleted." + else + echo + echo -e "Unable to connect to delete Elastalert indices. Exiting." + echo + exit 1 + fi + else + echo "Major Elasticsearch version is greater than 7...skipping Elastalert index maintenance." fi } From fbf0803906970653f5f403042198a3245622315f Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 18 Aug 2022 09:16:22 -0400 Subject: [PATCH 27/31] Update verbiage around major Elasticsearch version and not requiring Elastalert index maintenance --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 85ef432d1..8971e4371 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -457,7 +457,7 @@ elastalert_indices_check() { exit 1 fi else - echo "Major Elasticsearch version is greater than 7...skipping Elastalert index maintenance." + echo "Major Elasticsearch version is 8 or greater...skipping Elastalert index maintenance." fi } From fea2b481e33a5ff48b9dc8ab005a15605872691d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 19 Aug 2022 13:12:49 -0400 Subject: [PATCH 28/31] Update rulecat.conf --- salt/idstools/etc/rulecat.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/idstools/etc/rulecat.conf b/salt/idstools/etc/rulecat.conf index 2b1a8cae1..a799bba4b 100644 --- a/salt/idstools/etc/rulecat.conf +++ b/salt/idstools/etc/rulecat.conf @@ -31,11 +31,11 @@ {%- elif RULESET == 'ETPRO' %} --etpro={{ OINKCODE }} {%- elif RULESET == 'TALOS' %} ---url=https://www.snort.org/rules/snortrules-snapshot-2983.tar.gz?oinkcode={{ OINKCODE }} +--url=https://www.snort.org/rules/snortrules-snapshot-29200.tar.gz?oinkcode={{ OINKCODE }} {%- endif %} {%- endif %} {%- if URLS != None %} {%- for URL in URLS %} --url={{ URL }} {%- endfor %} -{%- endif %} \ No newline at end of file +{%- endif %} From f00d9074ffff7470a270172846df28d10aea1d1a Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 19 Aug 2022 16:07:14 -0400 Subject: [PATCH 29/31] Allow local modification acceptance prompt to be skipped when passing 'skip-prompt' as a parameter value to check_local_mods() function --- salt/common/tools/sbin/soup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 8971e4371..51eaafa52 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -227,13 +227,13 @@ check_local_mods() { echo "" echo "To reference this list later, check $SOUP_LOG". echo - if [[ -z $UNATTENDED ]]; then + if [[ -z $UNATTENDED ]] && ! [[ "${1}" == "skip-prompt" ]]; then while true; do read -p "Please review the local modifications shown above as they may cause problems during or after the update. Would you like to proceed with the update anyway? -If so, type 'YES'. Otherwise, type anything else to exit SOUP." yn +If so, type 'YES'. Otherwise, type anything else to exit SOUP. " yn case $yn in [yY][eE][sS] ) echo "Local modifications accepted. Continuing..."; break;; @@ -1405,7 +1405,7 @@ main() { fi echo "Checking for local modifications." - check_local_mods + check_local_mods skip-prompt echo "Checking sudoers file." check_sudoers From 33ebed34688d4b1c8ca74ddad0a6e85fcbf21393 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 22 Aug 2022 14:31:04 -0400 Subject: [PATCH 30/31] 2.3.150 --- VERIFY_ISO.md | 22 ++++++++++---------- sigs/securityonion-2.3.150-20220820.iso.sig | Bin 0 -> 543 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-2.3.150-20220820.iso.sig diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index cd5959ce8..b8555c3b2 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -1,18 +1,18 @@ -### 2.3.140-20220812 ISO image built on 2022/08/12 +### 2.3.150-20220820 ISO image built on 2022/08/12 ### Download and Verify -2.3.140-20220812 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.3.140-20220812.iso +2.3.150-20220820 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.3.150-20220820.iso -MD5: 13D4A5D663B5A36D045B980E5F33E6BC -SHA1: 85DC36B7E96575259DFD080BC860F6508D5F5899 -SHA256: DE5D0F82732B81456180AA40C124E5C82688611941EEAF03D85986806631588C +MD5: D2C0B67F19C18F0AB6FD1EC9B1E4034A +SHA1: F14BF42C6C634BDECA654B169FE6815BB6798F70 +SHA256: 9E37E5CCCBD209486EB79E8F991DE83F64E2208D32E5B56F8E0A6C3933EB42AC Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.140-20220812.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.150-20220820.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS @@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.140-20220812.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.150-20220820.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.3.140-20220812.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.3.150-20220820.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.3.140-20220812.iso.sig securityonion-2.3.140-20220812.iso +gpg --verify securityonion-2.3.150-20220820.iso.sig securityonion-2.3.150-20220820.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Fri 12 Aug 2022 03:59:11 PM EDT using RSA key ID FE507013 +gpg: Signature made Sat 20 Aug 2022 08:07:10 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.3.150-20220820.iso.sig b/sigs/securityonion-2.3.150-20220820.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..68a5a7a543c2c3871fa98eb0248c3bb3999cac80 GIT binary patch literal 543 zcmV+)0^t3L0vrSY0RjL91p;FMcP;=52@re`V7LBIa1*W)5BmkfBc)l{3YG*lH_;dw z$TS0UP_=C3V%Vs`NwiW=z##7x2Zi{6Ryc}Vzu&#Ji2(NB?&9ZE)ti&*y4v}st38tRim|&eG7z0%K8Oi;k4Tz z^5zPxZx(X;Cpuk^RJm+pV;~9ZLaL?AvP!~_mm1h9_a|8S2|=e&zP|(y{aHPJQi}vk zFRn<>GXnSY^HxqUiQ~T~HJu0~V18uhIXVO@ zg)D^%;=UL~uF2!4z_R(gEr1v-5zRpiv(7L6l71l%udD}7K6=Hvx2655>>`xyV)dqx zr)gHH*RhmBD1aOvGU&=xePws1sq@JocqsJFhpM*haa)3u<$ZU3 zrIpBbB^mrp%<=O7_vv^CEZ&dWgG+Lm1HkNG-W}hE3Dzuc7H?6rQnD;oyI>#LI`f5f hana1<6eb{L}*xkLIJ>DmVk7mZ%(y2QdHu literal 0 HcmV?d00001 From bd7b4c92bc7a70f6cd0ea86a263cf823a99f8899 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 22 Aug 2022 14:31:36 -0400 Subject: [PATCH 31/31] 2.3.150 --- VERIFY_ISO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index b8555c3b2..0ff07c6e3 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -1,4 +1,4 @@ -### 2.3.150-20220820 ISO image built on 2022/08/12 +### 2.3.150-20220820 ISO image built on 2022/08/20