From 839cfcaefa3e93ae0d20fb8aad7fca70813fcab2 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 2 Aug 2022 14:32:17 +0000 Subject: [PATCH 01/10] Update Elasticsearch defaults file and config.map.jinja to allow for local GeoIP database use when airgap is enabled --- salt/elasticsearch/config.map.jinja | 5 +++++ salt/elasticsearch/defaults.yaml | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 9a80ce30f..71ff5b912 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,5 +1,6 @@ {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} +{% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %} {% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} {% do ESCONFIG.elasticsearch.config.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} @@ -33,6 +34,10 @@ {% endif %} {% endif %} +{% if ISAIRGAP %} + {% do ESCONFIG.elasticsearch.config.ingest.geoip.downloader.update({'enabled': false}) %} +{% endif %} + {# merge with the elasticsearch pillar #} {% set ESCONFIG = salt['pillar.get']('elasticsearch:config', default=ESCONFIG.elasticsearch.config, merge=True) %} diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 830d1372c..782f2ad93 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -55,6 +55,10 @@ elasticsearch: indices: id_field_data: enabled: false + ingest: + geoip: + downloader: + enabled: true logger: org: elasticsearch: From c69cac0e5f7e66ed75ca93477ca2d748b6da2240 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 2 Aug 2022 11:31:35 -0400 Subject: [PATCH 02/10] Update Kibana version to 8.3.3 --- salt/kibana/bin/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index e19f25439..7b49f5a94 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -59,7 +59,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/8.3.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/8.3.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From f2b10a5a862b62a45449cd904c4499ad6aa27276 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 2 Aug 2022 11:32:01 -0400 Subject: [PATCH 03/10] Update Kibana version to 8.3.3 --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index 4ec8f9ca7..a8dc56f32 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.3.2","id": "8.3.2","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.3.3","id": "8.3.3","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From db8d9fff2c2f64c0fdf5ddeb5eb2172582aded36 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 2 Aug 2022 16:22:26 -0400 Subject: [PATCH 04/10] manage salt-minion start delay with systemd drop-in file - https://github.com/Security-Onion-Solutions/securityonion/issues/8441 --- salt/salt/map.jinja | 2 -- salt/salt/minion.sls | 6 +++--- salt/salt/service/salt-minion.service.jinja | 15 --------------- salt/salt/service/start-delay.conf.jinja | 2 ++ 4 files changed, 5 insertions(+), 20 deletions(-) delete mode 100644 salt/salt/service/salt-minion.service.jinja create mode 100644 salt/salt/service/start-delay.conf.jinja diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 389a95607..eb9f5ae89 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -11,7 +11,6 @@ {% set PYTHON3INFLUX= 'influxdb == ' ~ PYTHONINFLUXVERSION %} {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'python-dateutil', 'pytz', 'requests'] %} {% set PYTHONINSTALLER = 'pip' %} - {% set SYSTEMD_UNIT_FILE = '/lib/systemd/system/salt-minion.service' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %} @@ -22,7 +21,6 @@ {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} {% set PYTHONINSTALLER = 'pkg' %} - {% set SYSTEMD_UNIT_FILE = '/usr/lib/systemd/system/salt-minion.service' %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 15e203d82..fafb6f0f3 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -81,10 +81,10 @@ set_log_levels: - "log_level: error" - "log_level_logfile: error" -salt_minion_service_unit_file: +salt_minion_service_start_delay: file.managed: - - name: {{ SYSTEMD_UNIT_FILE }} - - source: salt://salt/service/salt-minion.service.jinja + - name: /etc/systemd/system/salt-minion.service.d/start-delay.conf + - source: salt://salt/service/start-delay.conf.jinja - template: jinja - defaults: service_start_delay: {{ service_start_delay }} diff --git a/salt/salt/service/salt-minion.service.jinja b/salt/salt/service/salt-minion.service.jinja deleted file mode 100644 index c7bae0bc2..000000000 --- a/salt/salt/service/salt-minion.service.jinja +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=The Salt Minion -Documentation=man:salt-minion(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltstack.com/en/latest/contents.html -After=network.target salt-master.service - -[Service] -KillMode=process -Type=notify -NotifyAccess=all -LimitNOFILE=8192 -ExecStart=/usr/bin/salt-minion -ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/salt/salt/service/start-delay.conf.jinja b/salt/salt/service/start-delay.conf.jinja new file mode 100644 index 000000000..33917b174 --- /dev/null +++ b/salt/salt/service/start-delay.conf.jinja @@ -0,0 +1,2 @@ +[Service] +ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }} From 9ac640fa67efc3f9dad307846111b517b24b23a2 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 3 Aug 2022 09:21:03 -0400 Subject: [PATCH 05/10] Remove airgap-specific logic for ingest.geoip.downloader --- salt/elasticsearch/config.map.jinja | 5 ----- 1 file changed, 5 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 71ff5b912..9a80ce30f 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,6 +1,5 @@ {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} -{% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %} {% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} {% do ESCONFIG.elasticsearch.config.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} @@ -34,10 +33,6 @@ {% endif %} {% endif %} -{% if ISAIRGAP %} - {% do ESCONFIG.elasticsearch.config.ingest.geoip.downloader.update({'enabled': false}) %} -{% endif %} - {# merge with the elasticsearch pillar #} {% set ESCONFIG = salt['pillar.get']('elasticsearch:config', default=ESCONFIG.elasticsearch.config, merge=True) %} From 8c694a7ca3f4b09f25f06e5d6eb2f2d3722bbf55 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 3 Aug 2022 09:21:40 -0400 Subject: [PATCH 06/10] Disable ingest.geoip.downloader by default --- salt/elasticsearch/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 782f2ad93..ee2051d12 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -58,7 +58,7 @@ elasticsearch: ingest: geoip: downloader: - enabled: true + enabled: false logger: org: elasticsearch: From 15f7fd892074d07827c77d085efc21f17d08f555 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 3 Aug 2022 15:16:12 -0400 Subject: [PATCH 07/10] fix the requisite --- salt/salt/minion.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index fafb6f0f3..fc6546392 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -109,7 +109,7 @@ salt_minion_service: - file: mine_functions {% if INSTALLEDSALTVERSION|string == SALTVERSION|string %} - file: set_log_levels - - file: salt_minion_service_unit_file + - file: salt_minion_service_start_delay {% endif %} - order: last From 99805cc326695d61196d76a6d99cd55d64a290f0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 3 Aug 2022 16:54:22 -0400 Subject: [PATCH 08/10] ensure parent dirs are created --- salt/salt/minion.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index fc6546392..7af96ff07 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -86,6 +86,7 @@ salt_minion_service_start_delay: - name: /etc/systemd/system/salt-minion.service.d/start-delay.conf - source: salt://salt/service/start-delay.conf.jinja - template: jinja + - makedirs: True - defaults: service_start_delay: {{ service_start_delay }} - onchanges_in: From f02f431dab2e7d6a712e8944084f4148d79783d5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 4 Aug 2022 16:34:06 -0400 Subject: [PATCH 09/10] ensure ExecStartPre is removed from default salt-minion service file --- salt/salt/minion.sls | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 7af96ff07..b3110a3f9 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -81,6 +81,14 @@ set_log_levels: - "log_level: error" - "log_level_logfile: error" +delete_pre_150_start_delay: + file.line: + - name: {{ SYSTEMD_UNIT_FILE }} + - match: ^ExecStartPre=* + - mode: delete + - onchanges_in: + - module: systemd_reload + salt_minion_service_start_delay: file.managed: - name: /etc/systemd/system/salt-minion.service.d/start-delay.conf From 3b62fc63c9dde9ad7c005046b0b6b580f31ea3e6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 5 Aug 2022 10:53:07 -0400 Subject: [PATCH 10/10] add SYSTEMD_UNIT_FILE back to map file --- salt/salt/map.jinja | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index eb9f5ae89..389a95607 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -11,6 +11,7 @@ {% set PYTHON3INFLUX= 'influxdb == ' ~ PYTHONINFLUXVERSION %} {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'python-dateutil', 'pytz', 'requests'] %} {% set PYTHONINSTALLER = 'pip' %} + {% set SYSTEMD_UNIT_FILE = '/lib/systemd/system/salt-minion.service' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %} @@ -21,6 +22,7 @@ {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} {% set PYTHONINSTALLER = 'pkg' %} + {% set SYSTEMD_UNIT_FILE = '/usr/lib/systemd/system/salt-minion.service' %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %}