From f7176f998912f071408a3e6c29515b26112f836f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 27 Feb 2023 09:58:43 -0500 Subject: [PATCH 01/18] Move Output to ES --- salt/common/tools/sbin/so-elastic-fleet-setup | 5 +++++ salt/elastic-fleet/install_agent_grid.sls | 2 +- salt/firewall/ports/ports.yaml | 1 + salt/top.sls | 10 ++++++++++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index b82890faa..b7bf1b3c3 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -84,6 +84,11 @@ printf '%s\n'\ " url: '{{ GLOBALS.manager_ip }}'"\ "" >> "$pillar_file" +#Store Grid Nodes Enrollment token in Global pillar +global_pillar_file=/opt/so/saltstack/local/pillar/soc_global.sls +printf '%s\n'\ + " fleet_grid_enrollment_token: '$GRIDNODESENROLLMENTOKEN'"\ + "" >> "$global_pillar_file" # Call Elastic-Fleet Salt State salt-call state.apply elastic-fleet queue=True diff --git a/salt/elastic-fleet/install_agent_grid.sls b/salt/elastic-fleet/install_agent_grid.sls index 857e31315..2f848ac2e 100644 --- a/salt/elastic-fleet/install_agent_grid.sls +++ b/salt/elastic-fleet/install_agent_grid.sls @@ -2,7 +2,7 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. -{%- set GRIDNODETOKEN = salt['pillar.get']('elasticfleet:server:grid_enrollment') -%} +{%- set GRIDNODETOKEN = salt['pillar.get']('global:fleet_grid_enrollment_token') -%} {% set AGENT_STATUS = salt['service.available']('elastic-agent') %} {% if not AGENT_STATUS %} diff --git a/salt/firewall/ports/ports.yaml b/salt/firewall/ports/ports.yaml index 22c6c8b09..de0539258 100644 --- a/salt/firewall/ports/ports.yaml +++ b/salt/firewall/ports/ports.yaml @@ -32,6 +32,7 @@ firewall: elastic_agent_data: tcp: - 5055 + - 9200 endgame: tcp: - 3765 diff --git a/salt/top.sls b/salt/top.sls index 25ec1ccd4..0a57f6e4e 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -63,6 +63,7 @@ base: {%- endif %} - schedule - docker_clean + - elastic-fleet.install_agent_grid '*_eval and G@saltversion:{{saltversion}}': - match: compound @@ -108,6 +109,7 @@ base: - playbook - redis {%- endif %} + - elastic-fleet.install_agent_grid - docker_clean '*_manager and G@saltversion:{{saltversion}}': @@ -148,6 +150,7 @@ base: - schedule - soctopus - playbook + - elastic-fleet.install_agent_grid - docker_clean '*_standalone and G@saltversion:{{saltversion}}': @@ -199,6 +202,7 @@ base: - playbook - docker_clean - elastic-fleet + - elastic-fleet.install_agent_grid '*_searchnode and G@saltversion:{{saltversion}}': - match: compound @@ -214,6 +218,7 @@ base: - logstash {%- endif %} - schedule + - elastic-fleet.install_agent_grid - docker_clean '*_managersearch and G@saltversion:{{saltversion}}': @@ -254,6 +259,7 @@ base: - schedule - soctopus - playbook + - elastic-fleet.install_agent_grid - docker_clean '*_heavynode and G@saltversion:{{saltversion}}': @@ -282,6 +288,7 @@ base: - zeek {%- endif %} - schedule + - elastic-fleet.install_agent_grid - docker_clean '*_import and G@saltversion:{{saltversion}}': @@ -312,6 +319,7 @@ base: - zeek - schedule - docker_clean + - elastic-fleet.install_agent_grid - elastic-fleet '*_receiver and G@saltversion:{{saltversion}}': @@ -327,6 +335,7 @@ base: - redis {%- endif %} - schedule + - elastic-fleet.install_agent_grid - docker_clean '*_idh and G@saltversion:{{saltversion}}': @@ -336,6 +345,7 @@ base: - telegraf - firewall - schedule + - elastic-fleet.install_agent_grid - docker_clean - idh From b4015ac73e7ed18f557bf06e94332f51c7b35697 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 27 Feb 2023 10:05:08 -0500 Subject: [PATCH 02/18] add sensor to node_containers --- salt/firewall/containers.map.jinja | 25 +++++++++++++------------ salt/firewall/init.sls | 10 ++++++++++ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index bd4cf72f1..a2114258f 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -22,9 +22,8 @@ 'so-strelka-manager', 'so-strelka-filestream' ] %} -{% endif %} -{% if GLOBALS.role == 'so-manager' or GLOBALS.role == 'so-standalone' or GLOBALS.role == 'so-managersearch' %} +{% elif GLOBALS.role == 'so-manager' or GLOBALS.role == 'so-standalone' or GLOBALS.role == 'so-managersearch' %} {% set NODE_CONTAINERS = [ 'so-curator', 'so-dockerregistry', @@ -47,17 +46,15 @@ 'so-strelka-manager', 'so-strelka-filestream' ] %} -{% endif %} -{% if GLOBALS.role == 'so-searchnode' %} +{% elif GLOBALS.role == 'so-searchnode' %} {% set NODE_CONTAINERS = [ 'so-elasticsearch', 'so-logstash', 'so-nginx' ] %} -{% endif %} -{% if GLOBALS.role == 'so-heavynode' %} +{% elif GLOBALS.role == 'so-heavynode' %} {% set NODE_CONTAINERS = [ 'so-curator', 'so-elasticsearch', @@ -71,9 +68,8 @@ 'so-strelka-manager', 'so-strelka-filestream' ] %} -{% endif %} -{% if GLOBALS.role == 'so-import' %} +{% elif GLOBALS.role == 'so-import' %} {% set NODE_CONTAINERS = [ 'so-dockerregistry', 'so-elasticsearch', @@ -85,17 +81,22 @@ 'so-nginx', 'so-soc' ] %} -{% endif %} -{% if GLOBALS.role == 'so-receiver' %} +{% elif GLOBALS.role == 'so-receiver' %} {% set NODE_CONTAINERS = [ 'so-logstash', 'so-redis', ] %} -{% endif %} -{% if GLOBALS.role == 'so-idh' %} +{% elif GLOBALS.role == 'so-idh' %} {% set NODE_CONTAINERS = [ 'so-idh', ] %} + +{% elif GLOBALS.role == 'so-sensor' %} +{% set NODE_CONTAINERS = [] %} + +{% else %} +{% set NODE_CONTAINERS = [] %} + {% endif %} diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index 12aaed39e..63e8d326b 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -5,6 +5,8 @@ disable_firewalld: service.dead: - name: firewalld - enable: False + - prereq: + - file: iptables_config create_sysconfig_iptables: file.touch: @@ -22,6 +24,14 @@ iptables_restore: cmd.run: - name: iptables-restore < /etc/sysconfig/iptables +enable_firewalld: + service.enabled: + - name: firewalld + - enable: True + - onfail: + - file: iptables_config + - cmd: iptables_restore + {% else %} {{sls}}_state_not_allowed: From 068d383442c165617bd497ae5a9e91aab15f3c12 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 27 Feb 2023 12:44:46 -0500 Subject: [PATCH 03/18] change to service.running --- salt/firewall/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index 63e8d326b..7feb8dd3b 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -25,7 +25,7 @@ iptables_restore: - name: iptables-restore < /etc/sysconfig/iptables enable_firewalld: - service.enabled: + service.running: - name: firewalld - enable: True - onfail: From 44ed48033cd77c67c811c8a8d7a3e649f95970d2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 27 Feb 2023 13:04:23 -0500 Subject: [PATCH 04/18] move requirement --- salt/firewall/init.sls | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index 7feb8dd3b..f59a39aca 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -1,13 +1,6 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -disable_firewalld: - service.dead: - - name: firewalld - - enable: False - - prereq: - - file: iptables_config - create_sysconfig_iptables: file.touch: - name: /etc/sysconfig/iptables @@ -20,6 +13,13 @@ iptables_config: - source: salt://firewall/iptables.jinja - template: jinja +disable_firewalld: + service.dead: + - name: firewalld + - enable: False + - require: + - file: iptables_config + iptables_restore: cmd.run: - name: iptables-restore < /etc/sysconfig/iptables From 9967e918255e37e453f721c74dd6afaa7d265bcf Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 27 Feb 2023 13:42:11 -0500 Subject: [PATCH 05/18] remove mysql check --- salt/mysql/init.sls | 2 -- setup/so-verify | 5 ++++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index c077695b8..8c4edc651 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -112,8 +112,6 @@ so-mysql: - onchanges: - docker_container: so-mysql module.run: - - so.mysql_conn: - - retry: 300 - onchanges: - cmd: so-mysql diff --git a/setup/so-verify b/setup/so-verify index 44c0465ff..f2b5a76b6 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -36,11 +36,14 @@ log_has_errors() { grep -vE "The Salt Master has cached the public key for this node" | \ grep -vE "Minion failed to authenticate with the master" | \ grep -vE "Failed to connect to ::1" | \ - grep -vE "Failed to set locale" | \ + grep -vE "Failed to set locale" | \ grep -vE "perl-Error-" | \ grep -vE "Failed:\s*?[0-9]+" | \ grep -vE "Status .* was not found" | \ grep -vE "Uncaught exception, closing connection" | \ + grep -vE "Server does not appear to be running or fully initialized - will try again in 10 seconds" | \ + grep -vE "Command 'so-influxdb-manage' failed with return code" | \ + grep -vE "retcode: 1" | \ grep -vE "Exception in callback None" | \ grep -vE "deprecation: ERROR" | \ grep -vE "code: 100" | \ From aa7b05d639d992c9c4e16e162ff3a67de49a49c2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 27 Feb 2023 14:12:26 -0500 Subject: [PATCH 06/18] small cleanup --- salt/allowed_states.map.jinja | 9 +-- salt/manager/files/acng/acng.conf | 96 ------------------------------- salt/mysql/init.sls | 14 ----- salt/top.sls | 4 -- salt/yum/packages.sls | 4 -- 5 files changed, 1 insertion(+), 126 deletions(-) delete mode 100644 salt/manager/files/acng/acng.conf delete mode 100644 salt/yum/packages.sls diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index bc2a7c86b..3548a7f0d 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -250,14 +250,7 @@ {% if REDIS and grains.role in ['so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-receiver'] %} {% do allowed_states.append('redis') %} {% endif %} - - {% if grains.os == 'Rocky' %} - {% if not ISAIRGAP %} - {% do allowed_states.append('yum') %} - {% endif %} - {% do allowed_states.append('yum.packages') %} - {% endif %} - + {# all nodes on the right salt version can run the following states #} {% do allowed_states.append('common') %} {% do allowed_states.append('patch.os.schedule') %} diff --git a/salt/manager/files/acng/acng.conf b/salt/manager/files/acng/acng.conf deleted file mode 100644 index 1f1f53754..000000000 --- a/salt/manager/files/acng/acng.conf +++ /dev/null @@ -1,96 +0,0 @@ -# This is a configuration file for apt-cacher-ng, a smart caching proxy for -CacheDir: /var/cache/apt-cacher-ng -LogDir: /var/log/apt-cacher-ng -Port: 3142 -# BindAddress: localhost 192.168.7.254 publicNameOnMainInterface -Remap-debrep: file:deb_mirror*.gz /debian ; file:backends_debian # Debian Archives -Remap-uburep: file:ubuntu_mirrors /ubuntu ; file:backends_ubuntu.us # Ubuntu Archives -Remap-cygwin: file:cygwin_mirrors /cygwin # ; file:backends_cygwin # incomplete, please create this file or specify preferred mirrors here -Remap-alxrep: file:archlx_mirrors /archlinux # ; file:backend_archlx # Arch Linux -Remap-centosmirrorlist: mirrorlist.centos.org -Remap-centos: file:centos_mirrors ; file:backends_centos.us # Fedora Linux -Remap-fedora: file:fedora_mirrors ; file:backends_fedora.us # Fedora Linux -Remap-epel: file:epel_mirrors ; file:backends_epel.us # Fedora EPEL -Remap-slrep: file:sl_mirrors # Scientific Linux -Remap-gentoo: file:gentoo_mirrors.gz /gentoo ; file:backends_gentoo # Gentoo Archives -#Remap-alpine: file:alpine_mirrors /alpine #; dl-cdn.alpinelinux.org # Alpine Archives -Remap-alpine: dl-cdn.alpinelinux.org -Remap-yarn: registry.yarnpkg.com -Remap-npm: registry.npmjs.org -Remap-node: nodejs.org -Remap-apache: file:apache_mirrors ; file:backends_apache.us -Remap-salt: repo.saltstack.com; https://repo.saltstack.com -Remap-securityonion: http://repocache.securityonion.net ; file:securityonion -# Remap-secdeb: security.debian.org -ReportPage: acng-report.html -# SocketPath:/var/run/apt-cacher-ng/socket -UnbufferLogs: 1 -VerboseLog: 1 -ForeGround: 1 -# PidFile: /var/run/apt-cacher-ng/pid -# Offlinemode: 0 -# ForceManaged: 0 -ExTreshold: 8 -# ExAbortOnProblems: 1 -# ExSuppressAdminNotification: 1 -# StupidFs: 0 -# ForwardBtsSoap: 1 -# DnsCacheSeconds: 1800 -# MaxStandbyConThreads: 8 -MaxConThreads: 120 -# -# - static data that doesn't change silently ont he server (PFilePattern) -# - volatile data that can be changed like every hour (VFilePattern) -# - special static data that shared some file names with volatile data, -# and in doubt should be identified as static (SPfilePattern) -# - a "whitelist pattern" with hints for the regular expiration job telling -# to keep the files even if they are not referenced by others, like crypto -# signatures with which clients begin their downloads (WfilePattern) -# -VfilePatternEx: (metalink\?repo=[0-9a-zA-Z-]+&arch=[0-9a-zA-Z_-]+|/\?release=[0-9]+&arch=|repodata/.*\.(xml|sqlite)\.(gz|bz2)|APKINDEX.tar.gz|filelists\.xml\.gz|filelists\.sqlite\.bz2|repomd\.xml|packages\.[a-zA-Z][a-zA-Z]\.gz) -PfilePatternEx: (/dists/.*/by-hash/.*|\.tgz|\.tar|\.xz|\.bz2|\.rpm|\.apk)$ -# WfilePatternEx: -# SPfilePatternEx: - -Debug:1 -# ExposeOrigin: 0 -# LogSubmittedOrigin: 0 -# UserAgent: Yet Another HTTP Client/1.2.3p4 -# RecompBz2: 0 -# NetworkTimeout: 60 - -# DontCacheRequested: linux-.*_10\...\.Custo._i386 -# DontCacheRequested: 192.168.0 ^10\..* 172.30 -# DontCacheResolved: ubuntumirror.local.net -DontCache: mirrorlist.centos.org - -# DirPerms: 00755 -# FilePerms: 00664 - -LocalDirs: acng-doc /usr/share/doc/apt-cacher-ng -# PrecacheFor: debrep/dists/unstable/*/source/Sources* debrep/dists/unstable/*/binary-amd64/Packages* -# RequestAppendix: X-Tracking-Choice: do-not-track\r\n -# ConnectProto: v6 v4 -# KeepExtraVersions: 0 -# UseWrap: 0 -FreshIndexMaxAge: 300 -# AllowUserPorts: 80 -RedirMax: 6 -# VfileUseRangeOps is set for fedora volatile files on mirrors that dont to range -VfileUseRangeOps: -1 -# PassThroughPattern: private-ppa\.launchpad\.net:443$ -# PassThroughPattern: .* # this would allow CONNECT to everything -PassThroughPattern: (repo\.securityonion\.net:443|download\.docker\.com:443|mirrors\.fedoraproject\.org:443|packages\.wazuh\.com:443|repo\.saltstack\.com:443|repo\.saltproject\.io:443|yum\.dockerproject\.org:443|download\.docker\.com:443|registry\.npmjs\.org:443|registry\.yarnpkg\.com:443)$ # yarn/npm pkg, cant to http :/ -# ResponseFreezeDetectTime: 500 -# ReuseConnections: 1 -# PipelineDepth: 255 -# CApath: /etc/ssl/certs -# CAfile: -# OptProxyTimeout: -1 -# MaxDlSpeed: 500 -# MaxInresponsiveDlSize: 64000 -# BadRedirDetectMime: text/html -{% set proxy = salt['pillar.get']('manager:proxy') -%} -{% if proxy -%} -Proxy: {{ proxy }} -{% endif -%} diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index 8c4edc651..44e6789af 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -106,20 +106,6 @@ so-mysql: - require: - file: mysqlcnf - file: mysqlpass - cmd.run: - - name: until nc -z {{ GLOBALS.manager }} 3306; do sleep 1; done - - timeout: 600 - - onchanges: - - docker_container: so-mysql - module.run: - - onchanges: - - cmd: so-mysql - -append_so-mysql_so-status.conf: - file.append: - - name: /opt/so/conf/so-status/so-status.conf - - text: so-mysql - {% endif %} {% else %} diff --git a/salt/top.sls b/salt/top.sls index c09974418..e029e3e2c 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -28,10 +28,6 @@ base: - salt.minion-state-apply-test - salt.minion - 'G@os:Rocky and G@saltversion:{{saltversion}}': - - match: compound - - yum.packages - '* and G@saltversion:{{saltversion}}': - match: compound - salt.minion diff --git a/salt/yum/packages.sls b/salt/yum/packages.sls deleted file mode 100644 index f70fa20be..000000000 --- a/salt/yum/packages.sls +++ /dev/null @@ -1,4 +0,0 @@ -install_yum_utils: - pkg.installed: - - name: yum-utils - From 8d33f01936f45b12d36b098f184e04df44a9f760 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 27 Feb 2023 15:01:31 -0500 Subject: [PATCH 07/18] Update defaults.map.jinja --- salt/soc/defaults.map.jinja | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.map.jinja b/salt/soc/defaults.map.jinja index 742d3d0dc..ff23914ec 100644 --- a/salt/soc/defaults.map.jinja +++ b/salt/soc/defaults.map.jinja @@ -2,7 +2,12 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER -%} {%- set INFLUXDB_TOKEN = salt['pillar.get']('secrets:influx_token') %} +{% if salt['docker.pid']('so-influxdb') > 0 %} {%- set METRICS_LINK = salt['cmd.run']('so-influxdb-manage dashboardpath "Security Onion Performance"') %} +{% else %} +{% set METRICS_LINK = '' %} +{% endif %} + {% for module, application_url in GLOBALS.application_urls.items() %} {% do SOCDEFAULTS.soc.server.modules[module].update({'hostUrl': application_url}) %} @@ -20,7 +25,7 @@ {% do SOCDEFAULTS.soc.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %} {% do SOCDEFAULTS.soc.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %} {% for tool in SOCDEFAULTS.soc.server.client.tools %} -{% if tool.name == "toolInfluxDb" %} +{% if tool.name == "toolInfluxDb" and METRICS_LINK | length > 0 %} {% do tool.update({'link': METRICS_LINK}) %} {% endif %} {% endfor %} From 16929707893e6b90cb1e22073197a845ef5f6e3f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 27 Feb 2023 15:22:08 -0500 Subject: [PATCH 08/18] back out verify changes since underlying errors should be fixed --- setup/so-verify | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup/so-verify b/setup/so-verify index f2b5a76b6..d9cbf9a88 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -41,9 +41,6 @@ log_has_errors() { grep -vE "Failed:\s*?[0-9]+" | \ grep -vE "Status .* was not found" | \ grep -vE "Uncaught exception, closing connection" | \ - grep -vE "Server does not appear to be running or fully initialized - will try again in 10 seconds" | \ - grep -vE "Command 'so-influxdb-manage' failed with return code" | \ - grep -vE "retcode: 1" | \ grep -vE "Exception in callback None" | \ grep -vE "deprecation: ERROR" | \ grep -vE "code: 100" | \ From 8632606a24ec1c2ef8b4510864f412da6d2ddb2f Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 27 Feb 2023 15:37:35 -0500 Subject: [PATCH 09/18] Update defaults.map.jinja --- salt/soc/defaults.map.jinja | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/salt/soc/defaults.map.jinja b/salt/soc/defaults.map.jinja index ff23914ec..eb9e57cb3 100644 --- a/salt/soc/defaults.map.jinja +++ b/salt/soc/defaults.map.jinja @@ -1,11 +1,13 @@ {% import_yaml 'soc/defaults.yaml' as SOCDEFAULTS %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER -%} -{%- set INFLUXDB_TOKEN = salt['pillar.get']('secrets:influx_token') %} -{% if salt['docker.pid']('so-influxdb') > 0 %} -{%- set METRICS_LINK = salt['cmd.run']('so-influxdb-manage dashboardpath "Security Onion Performance"') %} -{% else %} -{% set METRICS_LINK = '' %} +{% set INFLUXDB_TOKEN = salt['pillar.get']('secrets:influx_token') %} +{% if salt['docker.exists']('so-influxdb') %} +{% if salt['docker.pid']('so-influxdb') > 0 %} +{% set METRICS_LINK = salt['cmd.run']('so-influxdb-manage dashboardpath "Security Onion Performance"') %} +{% else %} +{% set METRICS_LINK = '' %} +{% endif %} {% endif %} From cbcd3c9dd9dff3456e933f66fbe200710122a11b Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 27 Feb 2023 15:39:03 -0500 Subject: [PATCH 10/18] Update defaults.map.jinja --- salt/soc/defaults.map.jinja | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/soc/defaults.map.jinja b/salt/soc/defaults.map.jinja index eb9e57cb3..95bfb2a76 100644 --- a/salt/soc/defaults.map.jinja +++ b/salt/soc/defaults.map.jinja @@ -2,11 +2,10 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER -%} {% set INFLUXDB_TOKEN = salt['pillar.get']('secrets:influx_token') %} +{% set METRICS_LINK = '' %} {% if salt['docker.exists']('so-influxdb') %} {% if salt['docker.pid']('so-influxdb') > 0 %} {% set METRICS_LINK = salt['cmd.run']('so-influxdb-manage dashboardpath "Security Onion Performance"') %} -{% else %} -{% set METRICS_LINK = '' %} {% endif %} {% endif %} From c00d6710988fa60ce907cc927cb81d1a76d45ebc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 28 Feb 2023 09:40:57 -0500 Subject: [PATCH 11/18] backup influx dir --- setup/so-functions | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-functions b/setup/so-functions index 381c4b497..35d625faa 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1832,6 +1832,7 @@ reinstall_init() { # Backup (and erase) directories in /nsm to prevent app errors backup_dir /nsm/mysql "$date_string" backup_dir /nsm/kratos "$date_string" + backup_dir /nsm/influxdb "$date_string" # Remove the old launcher package in case the config changes remove_package launcher-final From ff495cb20efc13e9341ad233459b9c875f362cae Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 28 Feb 2023 10:41:44 -0500 Subject: [PATCH 12/18] fix formatting issue --- setup/so-verify | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-verify b/setup/so-verify index d9cbf9a88..8c68852be 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -36,7 +36,7 @@ log_has_errors() { grep -vE "The Salt Master has cached the public key for this node" | \ grep -vE "Minion failed to authenticate with the master" | \ grep -vE "Failed to connect to ::1" | \ - grep -vE "Failed to set locale" | \ + grep -vE "Failed to set locale" | \ grep -vE "perl-Error-" | \ grep -vE "Failed:\s*?[0-9]+" | \ grep -vE "Status .* was not found" | \ From 052e0dea2eaaf698ec5098837af2db82797c1f8d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 28 Feb 2023 14:47:44 -0500 Subject: [PATCH 13/18] create and manage metrics_link in a file for soc --- salt/influxdb/init.sls | 8 ++++++++ salt/influxdb/metrics_link.txt | 0 salt/soc/defaults.map.jinja | 8 +------- 3 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 salt/influxdb/metrics_link.txt diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index f40b1b669..274502f46 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -117,6 +117,14 @@ influxdb-setup: - file: influxdb_curl_config - docker_container: so-influxdb +metrics_link_file: + file.managed: + - name: /opt/so/saltstack/local/salt/influxdb/metrics_link.txt + - contents: {{ salt['cmd.run']('so-influxdb-manage dashboardpath "Security Onion Performance"') }} + - require: + - docker_container: so-influxdb + - cmd: influxdb-setup + # Install cron job to determine size of influxdb for telegraf get_influxdb_size: cron.present: diff --git a/salt/influxdb/metrics_link.txt b/salt/influxdb/metrics_link.txt new file mode 100644 index 000000000..e69de29bb diff --git a/salt/soc/defaults.map.jinja b/salt/soc/defaults.map.jinja index 95bfb2a76..e16f047e4 100644 --- a/salt/soc/defaults.map.jinja +++ b/salt/soc/defaults.map.jinja @@ -2,13 +2,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER -%} {% set INFLUXDB_TOKEN = salt['pillar.get']('secrets:influx_token') %} -{% set METRICS_LINK = '' %} -{% if salt['docker.exists']('so-influxdb') %} -{% if salt['docker.pid']('so-influxdb') > 0 %} -{% set METRICS_LINK = salt['cmd.run']('so-influxdb-manage dashboardpath "Security Onion Performance"') %} -{% endif %} -{% endif %} - +{% import_text 'influxdb/metrics_link.txt' as METRICS_LINK %} {% for module, application_url in GLOBALS.application_urls.items() %} {% do SOCDEFAULTS.soc.server.modules[module].update({'hostUrl': application_url}) %} From 8772dcaa10bfd055b1b6efe81f6fc21a8377107e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 28 Feb 2023 15:57:54 -0500 Subject: [PATCH 14/18] ensure influxdb is running --- salt/influxdb/init.sls | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 274502f46..fb62cb323 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -117,13 +117,14 @@ influxdb-setup: - file: influxdb_curl_config - docker_container: so-influxdb +{% if salt['docker.exists']('so-influxdb') %} +{% if salt['docker.pid']('so-influxdb') > 0 %} metrics_link_file: file.managed: - name: /opt/so/saltstack/local/salt/influxdb/metrics_link.txt - contents: {{ salt['cmd.run']('so-influxdb-manage dashboardpath "Security Onion Performance"') }} - - require: - - docker_container: so-influxdb - - cmd: influxdb-setup +{% endif %} +{% endif %} # Install cron job to determine size of influxdb for telegraf get_influxdb_size: From 134caa7f58814c2ccc06e40e65eefefc49d1e28f Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 28 Feb 2023 16:31:16 -0500 Subject: [PATCH 15/18] Various adjustments to descriptions --- salt/elasticsearch/soc_elasticsearch.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 26d4890c2..05a812bbc 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -79,13 +79,13 @@ elasticsearch: phases: hot: min_age: - description: Minimum age + description: Minimum age of index. This determines when the index should be moved to the hot tier. global: True helpLink: elasticsearch.html actions: set_priority: priority: - description: Priority of index, used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. global: True helpLink: elasticsearch.html rollover: @@ -99,7 +99,7 @@ elasticsearch: helpLink: elasticsearch.html cold: min_age: - description: Minimum age of index, determining when it should be sent to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + description: Minimum age of index. This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. global: True helpLink: elasticsearch.html actions: @@ -110,7 +110,7 @@ elasticsearch: helpLink: elasticsearch.html delete: min_age: - description: Minimum age of index, determining when it should be deleted. + description: Minimum age of index. This determines when the index should be deleted. global: True helpLink: elastic so-endgame: *indexSettings From c4e1ec781eed1beef3d6ed8269267ed6ed360a50 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 28 Feb 2023 16:31:35 -0500 Subject: [PATCH 16/18] apply influxdb before highstate in setup --- setup/so-setup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 6825afc9d..e3221ed09 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -597,7 +597,8 @@ if ! [[ -f $install_opt_file ]]; then title "Seeding the docker registry" docker_seed_registry title "Applying the manager state" - logCmd "salt-call state.apply -l info manager" + logCmd "salt-call state.apply -l info manager" + logCmd "salt-call state.apply influxdb -l info" logCmd "salt-call state.highstate -l info" add_web_user info "Restarting SOC to pick up initial user" From a79c380e2b928e1ce13d79c4c3a99bc4af71fac5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 1 Mar 2023 09:18:58 -0500 Subject: [PATCH 17/18] use cmd.run to populate metrics_link --- salt/influxdb/init.sls | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index fb62cb323..2bc23792d 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -117,14 +117,11 @@ influxdb-setup: - file: influxdb_curl_config - docker_container: so-influxdb -{% if salt['docker.exists']('so-influxdb') %} -{% if salt['docker.pid']('so-influxdb') > 0 %} metrics_link_file: - file.managed: - - name: /opt/so/saltstack/local/salt/influxdb/metrics_link.txt - - contents: {{ salt['cmd.run']('so-influxdb-manage dashboardpath "Security Onion Performance"') }} -{% endif %} -{% endif %} + cmd.run: + - name: so-influxdb-manage dashboardpath "Security Onion Performance" &> /opt/so/saltstack/local/salt/influxdb/metrics_link.txt + - require: + - docker_container: so-influxdb # Install cron job to determine size of influxdb for telegraf get_influxdb_size: From 704365c6eb482362f9c97dced54a87cd3d955983 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 1 Mar 2023 09:44:48 -0500 Subject: [PATCH 18/18] only stdout redirect --- salt/influxdb/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 2bc23792d..bd894a6d4 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -119,7 +119,7 @@ influxdb-setup: metrics_link_file: cmd.run: - - name: so-influxdb-manage dashboardpath "Security Onion Performance" &> /opt/so/saltstack/local/salt/influxdb/metrics_link.txt + - name: so-influxdb-manage dashboardpath "Security Onion Performance" > /opt/so/saltstack/local/salt/influxdb/metrics_link.txt - require: - docker_container: so-influxdb