From a4897d20635a787ed0097fde88f2256af0c2a29e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 16 Dec 2020 09:07:38 -0500 Subject: [PATCH 01/11] [fix] Add Elasticsearch to containers running on Helix sensor --- salt/common/tools/sbin/so-image-common | 1 + salt/common/tools/sbin/soup | 11 ++++++----- salt/logstash/init.sls | 2 -- salt/top.sls | 1 + setup/so-setup | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 767f9d21c..01bb9727c 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -84,6 +84,7 @@ container_list() { TRUSTED_CONTAINERS=( "so-filebeat" "so-idstools" + "so-elasticsearch" "so-logstash" "so-nginx" "so-redis" diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 4d168c077..1c422280a 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -504,11 +504,12 @@ else update_docker_containers "soup" FEATURESCHECK=$(lookup_pillar features elastic) if [[ "$FEATURESCHECK" == "True" ]]; then - TRUSTED_CONTAINERS=( \ - "so-elasticsearch" \ - "so-filebeat" \ - "so-kibana" \ - "so-logstash" ) + TRUSTED_CONTAINERS=( + "so-elasticsearch" + "so-filebeat" + "so-kibana" + "so-logstash" + ) update_docker_containers "features" "-features" fi fi diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index d332f737a..e23e4eef2 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -45,10 +45,8 @@ {% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %} {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} -{% if grains['role'] != 'so-helix' %} include: - elasticsearch -{% endif %} # Create the logstash group logstashgroup: diff --git a/salt/top.sls b/salt/top.sls index b6913895d..18dd1b61a 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -61,6 +61,7 @@ base: - suricata - zeek - redis + - elasticsearch - logstash {%- if FILEBEAT %} - filebeat diff --git a/setup/so-setup b/setup/so-setup index 3c59c59cb..8300fe6ae 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -692,7 +692,7 @@ set_redirect >> $setup_log 2>&1 salt-call state.apply -l info nginx >> $setup_log 2>&1 fi - if [[ $is_manager || $is_node || $is_import ]]; then + if [[ $is_manager || $is_node || $is_import || $is_helix ]]; then set_progress_str 64 "$(print_salt_state_apply 'elasticsearch')" salt-call state.apply -l info elasticsearch >> $setup_log 2>&1 fi From af149d04a97602d082a3cc91633335d034c4f400 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 16 Dec 2020 09:18:40 -0500 Subject: [PATCH 02/11] [fix] Only run portions of ES state, do not run container --- salt/common/tools/sbin/so-image-common | 1 - salt/elasticsearch/init.sls | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 01bb9727c..767f9d21c 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -84,7 +84,6 @@ container_list() { TRUSTED_CONTAINERS=( "so-filebeat" "so-idstools" - "so-elasticsearch" "so-logstash" "so-nginx" "so-redis" diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 3e0bac708..fdd9b4565 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -86,6 +86,8 @@ capemz: - user: 939 - group: 939 +{% if grains['role'] != 'so-helix' %} + # Add ES Group elasticsearchgroup: group.present: @@ -251,10 +253,12 @@ so-elasticsearch-templates: - template: jinja {% endif %} +{% endif %} {# if grains['role'] != 'so-helix' #} + {% else %} elasticsearch_state_not_allowed: test.fail_without_changes: - name: elasticsearch_state_not_allowed -{% endif %} +{% endif %} {# if 'elasticsearch' in top_states #} From b8581366729cee6f2027af1336553bb5b8c37536 Mon Sep 17 00:00:00 2001 From: TOoSmOotH Date: Wed, 16 Dec 2020 09:24:59 -0500 Subject: [PATCH 03/11] Add jertel complaince --- salt/common/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 1c422280a..a87279a0c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -324,6 +324,8 @@ rc3_to_2.3.0() { fi + INSTALLEDVERSION=2.3.0 + } space_check() { From a1fc354a8957ae9146fa3e6c17732b30d3b0c4b9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 16 Dec 2020 09:32:32 -0500 Subject: [PATCH 04/11] [fix] Correct ordering of printf lines --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 2cf1b28cf..5f98e685e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -996,8 +996,8 @@ fireeye_pillar() { printf '%s\n'\ "fireeye:"\ " helix:"\ - "" > "$fireeye_pillar_path"/init.sls " api_key: '$HELIXAPIKEY'" \ + "" > "$fireeye_pillar_path/init.sls" } From c68b87db566021d35e6624c53501a4fed6f46be1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 16 Dec 2020 09:33:44 -0500 Subject: [PATCH 05/11] set steno running default based on sensor role or not --- salt/sensoroni/files/sensoroni.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 55b928ef0..2e64dd2a6 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -3,7 +3,13 @@ {% set ADDRESS = salt['pillar.get']('sensoroni:node_address') -%} {% set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') -%} {% set CHECKININTERVALMS = salt['pillar.get']('sensoroni:node_checkin_interval_ms', 10000) -%} -{% set STENOENABLED = salt['pillar.get']('steno:enabled', False) -%} +{%- set ROLE = grains.id.split('_') | last %} +{%- if ROLE in ['eval', 'standalone', 'sensor', 'heavynode'] %} +{%- set STENODEFAULT = True %} +{%- else %} +{%- set STENODEFAULT = False %} +{%- endif } +{%- set STENOENABLED = salt['pillar.get']('steno:enabled', STENODEFAULT) %} { "logFilename": "/opt/sensoroni/logs/sensoroni.log", "logLevel":"info", From f0999abd8ec78481f4ac15ea149eded2dc646098 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 16 Dec 2020 09:38:21 -0500 Subject: [PATCH 06/11] add missing % --- salt/sensoroni/files/sensoroni.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 2e64dd2a6..ac4762b12 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -8,7 +8,7 @@ {%- set STENODEFAULT = True %} {%- else %} {%- set STENODEFAULT = False %} -{%- endif } +{%- endif %} {%- set STENOENABLED = salt['pillar.get']('steno:enabled', STENODEFAULT) %} { "logFilename": "/opt/sensoroni/logs/sensoroni.log", From 448d0e079eca4bc504ecffa42c38abd7cf551450 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 16 Dec 2020 09:39:25 -0500 Subject: [PATCH 07/11] add whitespace removal to the front --- salt/sensoroni/files/sensoroni.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index ac4762b12..23b967b04 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -1,8 +1,8 @@ -{% set URLBASE = salt['pillar.get']('global:url_base') -%} -{% set DESCRIPTION = salt['pillar.get']('sensoroni:node_description') -%} -{% set ADDRESS = salt['pillar.get']('sensoroni:node_address') -%} -{% set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') -%} -{% set CHECKININTERVALMS = salt['pillar.get']('sensoroni:node_checkin_interval_ms', 10000) -%} +{%- set URLBASE = salt['pillar.get']('global:url_base') %} +{%- set DESCRIPTION = salt['pillar.get']('sensoroni:node_description') %} +{%- set ADDRESS = salt['pillar.get']('sensoroni:node_address') %} +{%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') %} +{%- set CHECKININTERVALMS = salt['pillar.get']('sensoroni:node_checkin_interval_ms', 10000) %} {%- set ROLE = grains.id.split('_') | last %} {%- if ROLE in ['eval', 'standalone', 'sensor', 'heavynode'] %} {%- set STENODEFAULT = True %} From 8889c79afdbbd019e87107ba72e6b796ec9b81c9 Mon Sep 17 00:00:00 2001 From: TOoSmOotH Date: Wed, 16 Dec 2020 09:39:41 -0500 Subject: [PATCH 08/11] Run a common state first to fix docker race condition --- salt/common/tools/sbin/soup | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index a87279a0c..f9ac6de2b 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -583,6 +583,9 @@ if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then echo "" fi +echo "" +echo "Applying common state for any package updates." +salt-call -l info state.apply common queue=True echo "" echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." salt-call state.highstate -l info queue=True From aa0d43b1db87d574f9b56a9f09282c37aa5aa9b2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 16 Dec 2020 09:55:03 -0500 Subject: [PATCH 09/11] [fix] Always define ismanager var --- salt/elasticsearch/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index fdd9b4565..eb8f281b5 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -38,6 +38,8 @@ {% set esclustername = salt['pillar.get']('elasticsearch:esclustername') %} {% set esheap = salt['pillar.get']('elasticsearch:esheap') %} {% set ismanager = False %} +{% else %} + {% set ismanager = False %} {% endif %} {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} From e464117e8a3951242f26f86d5d9c144246ff34d6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 16 Dec 2020 10:19:44 -0500 Subject: [PATCH 10/11] [fix] Run so-catrust in ES state on Helix sensor install --- salt/elasticsearch/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index eb8f281b5..4ebe05cec 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -38,8 +38,8 @@ {% set esclustername = salt['pillar.get']('elasticsearch:esclustername') %} {% set esheap = salt['pillar.get']('elasticsearch:esheap') %} {% set ismanager = False %} -{% else %} - {% set ismanager = False %} +{% elif grains['role'] = 'so-helix' %} + {% set ismanager = True %} {# Solely for the sake of running so-catrust #} {% endif %} {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} From 142649b396b0b7b9198dc1cadd175389eb777355 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 16 Dec 2020 10:38:34 -0500 Subject: [PATCH 11/11] [fix] Fix comparator --- salt/elasticsearch/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 4ebe05cec..300921807 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -38,7 +38,7 @@ {% set esclustername = salt['pillar.get']('elasticsearch:esclustername') %} {% set esheap = salt['pillar.get']('elasticsearch:esheap') %} {% set ismanager = False %} -{% elif grains['role'] = 'so-helix' %} +{% elif grains['role'] == 'so-helix' %} {% set ismanager = True %} {# Solely for the sake of running so-catrust #} {% endif %}