diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index bc2a7c86b..3548a7f0d 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -250,14 +250,7 @@ {% if REDIS and grains.role in ['so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-receiver'] %} {% do allowed_states.append('redis') %} {% endif %} - - {% if grains.os == 'Rocky' %} - {% if not ISAIRGAP %} - {% do allowed_states.append('yum') %} - {% endif %} - {% do allowed_states.append('yum.packages') %} - {% endif %} - + {# all nodes on the right salt version can run the following states #} {% do allowed_states.append('common') %} {% do allowed_states.append('patch.os.schedule') %} diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index b82890faa..b7bf1b3c3 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -84,6 +84,11 @@ printf '%s\n'\ " url: '{{ GLOBALS.manager_ip }}'"\ "" >> "$pillar_file" +#Store Grid Nodes Enrollment token in Global pillar +global_pillar_file=/opt/so/saltstack/local/pillar/soc_global.sls +printf '%s\n'\ + " fleet_grid_enrollment_token: '$GRIDNODESENROLLMENTOKEN'"\ + "" >> "$global_pillar_file" # Call Elastic-Fleet Salt State salt-call state.apply elastic-fleet queue=True diff --git a/salt/elastic-fleet/install_agent_grid.sls b/salt/elastic-fleet/install_agent_grid.sls index 857e31315..2f848ac2e 100644 --- a/salt/elastic-fleet/install_agent_grid.sls +++ b/salt/elastic-fleet/install_agent_grid.sls @@ -2,7 +2,7 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. -{%- set GRIDNODETOKEN = salt['pillar.get']('elasticfleet:server:grid_enrollment') -%} +{%- set GRIDNODETOKEN = salt['pillar.get']('global:fleet_grid_enrollment_token') -%} {% set AGENT_STATUS = salt['service.available']('elastic-agent') %} {% if not AGENT_STATUS %} diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 26d4890c2..05a812bbc 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -79,13 +79,13 @@ elasticsearch: phases: hot: min_age: - description: Minimum age + description: Minimum age of index. This determines when the index should be moved to the hot tier. global: True helpLink: elasticsearch.html actions: set_priority: priority: - description: Priority of index, used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. global: True helpLink: elasticsearch.html rollover: @@ -99,7 +99,7 @@ elasticsearch: helpLink: elasticsearch.html cold: min_age: - description: Minimum age of index, determining when it should be sent to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + description: Minimum age of index. This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. global: True helpLink: elasticsearch.html actions: @@ -110,7 +110,7 @@ elasticsearch: helpLink: elasticsearch.html delete: min_age: - description: Minimum age of index, determining when it should be deleted. + description: Minimum age of index. This determines when the index should be deleted. global: True helpLink: elastic so-endgame: *indexSettings diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index bd4cf72f1..a2114258f 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -22,9 +22,8 @@ 'so-strelka-manager', 'so-strelka-filestream' ] %} -{% endif %} -{% if GLOBALS.role == 'so-manager' or GLOBALS.role == 'so-standalone' or GLOBALS.role == 'so-managersearch' %} +{% elif GLOBALS.role == 'so-manager' or GLOBALS.role == 'so-standalone' or GLOBALS.role == 'so-managersearch' %} {% set NODE_CONTAINERS = [ 'so-curator', 'so-dockerregistry', @@ -47,17 +46,15 @@ 'so-strelka-manager', 'so-strelka-filestream' ] %} -{% endif %} -{% if GLOBALS.role == 'so-searchnode' %} +{% elif GLOBALS.role == 'so-searchnode' %} {% set NODE_CONTAINERS = [ 'so-elasticsearch', 'so-logstash', 'so-nginx' ] %} -{% endif %} -{% if GLOBALS.role == 'so-heavynode' %} +{% elif GLOBALS.role == 'so-heavynode' %} {% set NODE_CONTAINERS = [ 'so-curator', 'so-elasticsearch', @@ -71,9 +68,8 @@ 'so-strelka-manager', 'so-strelka-filestream' ] %} -{% endif %} -{% if GLOBALS.role == 'so-import' %} +{% elif GLOBALS.role == 'so-import' %} {% set NODE_CONTAINERS = [ 'so-dockerregistry', 'so-elasticsearch', @@ -85,17 +81,22 @@ 'so-nginx', 'so-soc' ] %} -{% endif %} -{% if GLOBALS.role == 'so-receiver' %} +{% elif GLOBALS.role == 'so-receiver' %} {% set NODE_CONTAINERS = [ 'so-logstash', 'so-redis', ] %} -{% endif %} -{% if GLOBALS.role == 'so-idh' %} +{% elif GLOBALS.role == 'so-idh' %} {% set NODE_CONTAINERS = [ 'so-idh', ] %} + +{% elif GLOBALS.role == 'so-sensor' %} +{% set NODE_CONTAINERS = [] %} + +{% else %} +{% set NODE_CONTAINERS = [] %} + {% endif %} diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index 12aaed39e..f59a39aca 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -1,11 +1,6 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -disable_firewalld: - service.dead: - - name: firewalld - - enable: False - create_sysconfig_iptables: file.touch: - name: /etc/sysconfig/iptables @@ -18,10 +13,25 @@ iptables_config: - source: salt://firewall/iptables.jinja - template: jinja +disable_firewalld: + service.dead: + - name: firewalld + - enable: False + - require: + - file: iptables_config + iptables_restore: cmd.run: - name: iptables-restore < /etc/sysconfig/iptables +enable_firewalld: + service.running: + - name: firewalld + - enable: True + - onfail: + - file: iptables_config + - cmd: iptables_restore + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/firewall/ports/ports.yaml b/salt/firewall/ports/ports.yaml index 22c6c8b09..de0539258 100644 --- a/salt/firewall/ports/ports.yaml +++ b/salt/firewall/ports/ports.yaml @@ -32,6 +32,7 @@ firewall: elastic_agent_data: tcp: - 5055 + - 9200 endgame: tcp: - 3765 diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index f40b1b669..bd894a6d4 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -117,6 +117,12 @@ influxdb-setup: - file: influxdb_curl_config - docker_container: so-influxdb +metrics_link_file: + cmd.run: + - name: so-influxdb-manage dashboardpath "Security Onion Performance" > /opt/so/saltstack/local/salt/influxdb/metrics_link.txt + - require: + - docker_container: so-influxdb + # Install cron job to determine size of influxdb for telegraf get_influxdb_size: cron.present: diff --git a/salt/influxdb/metrics_link.txt b/salt/influxdb/metrics_link.txt new file mode 100644 index 000000000..e69de29bb diff --git a/salt/manager/files/acng/acng.conf b/salt/manager/files/acng/acng.conf deleted file mode 100644 index 1f1f53754..000000000 --- a/salt/manager/files/acng/acng.conf +++ /dev/null @@ -1,96 +0,0 @@ -# This is a configuration file for apt-cacher-ng, a smart caching proxy for -CacheDir: /var/cache/apt-cacher-ng -LogDir: /var/log/apt-cacher-ng -Port: 3142 -# BindAddress: localhost 192.168.7.254 publicNameOnMainInterface -Remap-debrep: file:deb_mirror*.gz /debian ; file:backends_debian # Debian Archives -Remap-uburep: file:ubuntu_mirrors /ubuntu ; file:backends_ubuntu.us # Ubuntu Archives -Remap-cygwin: file:cygwin_mirrors /cygwin # ; file:backends_cygwin # incomplete, please create this file or specify preferred mirrors here -Remap-alxrep: file:archlx_mirrors /archlinux # ; file:backend_archlx # Arch Linux -Remap-centosmirrorlist: mirrorlist.centos.org -Remap-centos: file:centos_mirrors ; file:backends_centos.us # Fedora Linux -Remap-fedora: file:fedora_mirrors ; file:backends_fedora.us # Fedora Linux -Remap-epel: file:epel_mirrors ; file:backends_epel.us # Fedora EPEL -Remap-slrep: file:sl_mirrors # Scientific Linux -Remap-gentoo: file:gentoo_mirrors.gz /gentoo ; file:backends_gentoo # Gentoo Archives -#Remap-alpine: file:alpine_mirrors /alpine #; dl-cdn.alpinelinux.org # Alpine Archives -Remap-alpine: dl-cdn.alpinelinux.org -Remap-yarn: registry.yarnpkg.com -Remap-npm: registry.npmjs.org -Remap-node: nodejs.org -Remap-apache: file:apache_mirrors ; file:backends_apache.us -Remap-salt: repo.saltstack.com; https://repo.saltstack.com -Remap-securityonion: http://repocache.securityonion.net ; file:securityonion -# Remap-secdeb: security.debian.org -ReportPage: acng-report.html -# SocketPath:/var/run/apt-cacher-ng/socket -UnbufferLogs: 1 -VerboseLog: 1 -ForeGround: 1 -# PidFile: /var/run/apt-cacher-ng/pid -# Offlinemode: 0 -# ForceManaged: 0 -ExTreshold: 8 -# ExAbortOnProblems: 1 -# ExSuppressAdminNotification: 1 -# StupidFs: 0 -# ForwardBtsSoap: 1 -# DnsCacheSeconds: 1800 -# MaxStandbyConThreads: 8 -MaxConThreads: 120 -# -# - static data that doesn't change silently ont he server (PFilePattern) -# - volatile data that can be changed like every hour (VFilePattern) -# - special static data that shared some file names with volatile data, -# and in doubt should be identified as static (SPfilePattern) -# - a "whitelist pattern" with hints for the regular expiration job telling -# to keep the files even if they are not referenced by others, like crypto -# signatures with which clients begin their downloads (WfilePattern) -# -VfilePatternEx: (metalink\?repo=[0-9a-zA-Z-]+&arch=[0-9a-zA-Z_-]+|/\?release=[0-9]+&arch=|repodata/.*\.(xml|sqlite)\.(gz|bz2)|APKINDEX.tar.gz|filelists\.xml\.gz|filelists\.sqlite\.bz2|repomd\.xml|packages\.[a-zA-Z][a-zA-Z]\.gz) -PfilePatternEx: (/dists/.*/by-hash/.*|\.tgz|\.tar|\.xz|\.bz2|\.rpm|\.apk)$ -# WfilePatternEx: -# SPfilePatternEx: - -Debug:1 -# ExposeOrigin: 0 -# LogSubmittedOrigin: 0 -# UserAgent: Yet Another HTTP Client/1.2.3p4 -# RecompBz2: 0 -# NetworkTimeout: 60 - -# DontCacheRequested: linux-.*_10\...\.Custo._i386 -# DontCacheRequested: 192.168.0 ^10\..* 172.30 -# DontCacheResolved: ubuntumirror.local.net -DontCache: mirrorlist.centos.org - -# DirPerms: 00755 -# FilePerms: 00664 - -LocalDirs: acng-doc /usr/share/doc/apt-cacher-ng -# PrecacheFor: debrep/dists/unstable/*/source/Sources* debrep/dists/unstable/*/binary-amd64/Packages* -# RequestAppendix: X-Tracking-Choice: do-not-track\r\n -# ConnectProto: v6 v4 -# KeepExtraVersions: 0 -# UseWrap: 0 -FreshIndexMaxAge: 300 -# AllowUserPorts: 80 -RedirMax: 6 -# VfileUseRangeOps is set for fedora volatile files on mirrors that dont to range -VfileUseRangeOps: -1 -# PassThroughPattern: private-ppa\.launchpad\.net:443$ -# PassThroughPattern: .* # this would allow CONNECT to everything -PassThroughPattern: (repo\.securityonion\.net:443|download\.docker\.com:443|mirrors\.fedoraproject\.org:443|packages\.wazuh\.com:443|repo\.saltstack\.com:443|repo\.saltproject\.io:443|yum\.dockerproject\.org:443|download\.docker\.com:443|registry\.npmjs\.org:443|registry\.yarnpkg\.com:443)$ # yarn/npm pkg, cant to http :/ -# ResponseFreezeDetectTime: 500 -# ReuseConnections: 1 -# PipelineDepth: 255 -# CApath: /etc/ssl/certs -# CAfile: -# OptProxyTimeout: -1 -# MaxDlSpeed: 500 -# MaxInresponsiveDlSize: 64000 -# BadRedirDetectMime: text/html -{% set proxy = salt['pillar.get']('manager:proxy') -%} -{% if proxy -%} -Proxy: {{ proxy }} -{% endif -%} diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index c077695b8..44e6789af 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -106,22 +106,6 @@ so-mysql: - require: - file: mysqlcnf - file: mysqlpass - cmd.run: - - name: until nc -z {{ GLOBALS.manager }} 3306; do sleep 1; done - - timeout: 600 - - onchanges: - - docker_container: so-mysql - module.run: - - so.mysql_conn: - - retry: 300 - - onchanges: - - cmd: so-mysql - -append_so-mysql_so-status.conf: - file.append: - - name: /opt/so/conf/so-status/so-status.conf - - text: so-mysql - {% endif %} {% else %} diff --git a/salt/soc/defaults.map.jinja b/salt/soc/defaults.map.jinja index 742d3d0dc..e16f047e4 100644 --- a/salt/soc/defaults.map.jinja +++ b/salt/soc/defaults.map.jinja @@ -1,8 +1,8 @@ {% import_yaml 'soc/defaults.yaml' as SOCDEFAULTS %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER -%} -{%- set INFLUXDB_TOKEN = salt['pillar.get']('secrets:influx_token') %} -{%- set METRICS_LINK = salt['cmd.run']('so-influxdb-manage dashboardpath "Security Onion Performance"') %} +{% set INFLUXDB_TOKEN = salt['pillar.get']('secrets:influx_token') %} +{% import_text 'influxdb/metrics_link.txt' as METRICS_LINK %} {% for module, application_url in GLOBALS.application_urls.items() %} {% do SOCDEFAULTS.soc.server.modules[module].update({'hostUrl': application_url}) %} @@ -20,7 +20,7 @@ {% do SOCDEFAULTS.soc.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %} {% do SOCDEFAULTS.soc.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %} {% for tool in SOCDEFAULTS.soc.server.client.tools %} -{% if tool.name == "toolInfluxDb" %} +{% if tool.name == "toolInfluxDb" and METRICS_LINK | length > 0 %} {% do tool.update({'link': METRICS_LINK}) %} {% endif %} {% endfor %} diff --git a/salt/top.sls b/salt/top.sls index c09974418..69d99f6de 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -28,10 +28,6 @@ base: - salt.minion-state-apply-test - salt.minion - 'G@os:Rocky and G@saltversion:{{saltversion}}': - - match: compound - - yum.packages - '* and G@saltversion:{{saltversion}}': - match: compound - salt.minion @@ -63,6 +59,7 @@ base: {%- endif %} - schedule - docker_clean + - elastic-fleet.install_agent_grid '*_eval and G@saltversion:{{saltversion}}': - match: compound @@ -109,6 +106,7 @@ base: - playbook - redis {%- endif %} + - elastic-fleet.install_agent_grid - docker_clean '*_manager and G@saltversion:{{saltversion}}': @@ -150,6 +148,7 @@ base: - schedule - soctopus - playbook + - elastic-fleet.install_agent_grid - docker_clean '*_standalone and G@saltversion:{{saltversion}}': @@ -202,6 +201,7 @@ base: - playbook - docker_clean - elastic-fleet + - elastic-fleet.install_agent_grid '*_searchnode and G@saltversion:{{saltversion}}': - match: compound @@ -217,6 +217,7 @@ base: - logstash {%- endif %} - schedule + - elastic-fleet.install_agent_grid - docker_clean '*_managersearch and G@saltversion:{{saltversion}}': @@ -258,6 +259,7 @@ base: - schedule - soctopus - playbook + - elastic-fleet.install_agent_grid - docker_clean '*_heavynode and G@saltversion:{{saltversion}}': @@ -286,6 +288,7 @@ base: - zeek {%- endif %} - schedule + - elastic-fleet.install_agent_grid - docker_clean '*_import and G@saltversion:{{saltversion}}': @@ -317,6 +320,7 @@ base: - zeek - schedule - docker_clean + - elastic-fleet.install_agent_grid - elastic-fleet '*_receiver and G@saltversion:{{saltversion}}': @@ -332,6 +336,7 @@ base: - redis {%- endif %} - schedule + - elastic-fleet.install_agent_grid - docker_clean '*_idh and G@saltversion:{{saltversion}}': @@ -341,6 +346,7 @@ base: - telegraf - firewall - schedule + - elastic-fleet.install_agent_grid - docker_clean - idh diff --git a/salt/yum/packages.sls b/salt/yum/packages.sls deleted file mode 100644 index f70fa20be..000000000 --- a/salt/yum/packages.sls +++ /dev/null @@ -1,4 +0,0 @@ -install_yum_utils: - pkg.installed: - - name: yum-utils - diff --git a/setup/so-functions b/setup/so-functions index d044bf326..74913c32f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1832,6 +1832,7 @@ reinstall_init() { # Backup (and erase) directories in /nsm to prevent app errors backup_dir /nsm/mysql "$date_string" backup_dir /nsm/kratos "$date_string" + backup_dir /nsm/influxdb "$date_string" # Remove the old launcher package in case the config changes remove_package launcher-final diff --git a/setup/so-setup b/setup/so-setup index 440965ca1..1bee9e617 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -611,7 +611,8 @@ if ! [[ -f $install_opt_file ]]; then title "Seeding the docker registry" docker_seed_registry title "Applying the manager state" - logCmd "salt-call state.apply -l info manager" + logCmd "salt-call state.apply -l info manager" + logCmd "salt-call state.apply influxdb -l info" logCmd "salt-call state.highstate -l info" add_web_user info "Restarting SOC to pick up initial user" diff --git a/setup/so-verify b/setup/so-verify index 44c0465ff..8c68852be 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -36,7 +36,7 @@ log_has_errors() { grep -vE "The Salt Master has cached the public key for this node" | \ grep -vE "Minion failed to authenticate with the master" | \ grep -vE "Failed to connect to ::1" | \ - grep -vE "Failed to set locale" | \ + grep -vE "Failed to set locale" | \ grep -vE "perl-Error-" | \ grep -vE "Failed:\s*?[0-9]+" | \ grep -vE "Status .* was not found" | \