Merge branch '2.4/dev' into kilo

2025-12-06 09:12:45 +01:00 · 2023-03-01 09:53:59 -05:00
parent 13182fcda2 2091806f1f
commit 41bc1cd36f
17 changed files with 63 additions and 155 deletions
--- a/salt/allowed_states.map.jinja
+++ b/salt/allowed_states.map.jinja
@@ -250,14 +250,7 @@
  {% if REDIS and grains.role in ['so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
    {% do allowed_states.append('redis') %}
  {% endif %}
-
-  {% if grains.os == 'Rocky' %}
-    {% if not ISAIRGAP %}
-      {% do allowed_states.append('yum') %}
-    {% endif %}
-    {% do allowed_states.append('yum.packages') %}
-  {% endif %}
-
+  
  {# all nodes on the right salt version can run the following states #}
  {% do allowed_states.append('common') %}
  {% do allowed_states.append('patch.os.schedule') %}
--- a/salt/common/tools/sbin/so-elastic-fleet-setup
+++ b/salt/common/tools/sbin/so-elastic-fleet-setup
@@ -84,6 +84,11 @@ printf '%s\n'\
    "    url: '{{ GLOBALS.manager_ip }}'"\
    "" >> "$pillar_file"

+#Store Grid Nodes Enrollment token in Global pillar
+global_pillar_file=/opt/so/saltstack/local/pillar/soc_global.sls
+printf '%s\n'\
+    "  fleet_grid_enrollment_token: '$GRIDNODESENROLLMENTOKEN'"\
+    "" >> "$global_pillar_file"

 # Call Elastic-Fleet Salt State
 salt-call state.apply elastic-fleet queue=True
--- a/salt/elastic-fleet/install_agent_grid.sls
+++ b/salt/elastic-fleet/install_agent_grid.sls
@@ -2,7 +2,7 @@
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.

-{%- set GRIDNODETOKEN = salt['pillar.get']('elasticfleet:server:grid_enrollment') -%}
+{%- set GRIDNODETOKEN = salt['pillar.get']('global:fleet_grid_enrollment_token') -%}

 {% set AGENT_STATUS = salt['service.available']('elastic-agent') %}
 {% if not AGENT_STATUS  %}
--- a/salt/elasticsearch/soc_elasticsearch.yaml
+++ b/salt/elasticsearch/soc_elasticsearch.yaml
@@ -79,13 +79,13 @@ elasticsearch:
        phases:
          hot:
            min_age:
-              description: Minimum age
+              description: Minimum age of index. This determines when the index should be moved to the hot tier.
              global: True
              helpLink: elasticsearch.html
            actions:
              set_priority:
                priority:
-                  description: Priority of index, used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
+                  description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
                  global: True
                  helpLink: elasticsearch.html
              rollover:
@@ -99,7 +99,7 @@ elasticsearch:
                  helpLink: elasticsearch.html
          cold:
            min_age:
-              description: Minimum age of index, determining when it should be sent to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
+              description: Minimum age of index.  This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
              global: True
              helpLink: elasticsearch.html
            actions:
@@ -110,7 +110,7 @@ elasticsearch:
                  helpLink: elasticsearch.html
          delete:
            min_age:
-              description: Minimum age of index, determining when it should be deleted.
+              description: Minimum age of index. This determines when the index should be deleted.
              global: True
              helpLink: elastic
    so-endgame: *indexSettings
--- a/salt/firewall/containers.map.jinja
+++ b/salt/firewall/containers.map.jinja
@@ -22,9 +22,8 @@
     'so-strelka-manager',
     'so-strelka-filestream'
 ] %}
-{% endif %}

-{% if GLOBALS.role == 'so-manager' or GLOBALS.role == 'so-standalone' or GLOBALS.role == 'so-managersearch' %}
+{% elif GLOBALS.role == 'so-manager' or GLOBALS.role == 'so-standalone' or GLOBALS.role == 'so-managersearch' %}
 {% set NODE_CONTAINERS = [
     'so-curator',
     'so-dockerregistry',
@@ -47,17 +46,15 @@
     'so-strelka-manager',
     'so-strelka-filestream'
 ] %}
-{% endif %}

-{% if GLOBALS.role == 'so-searchnode' %}
+{% elif GLOBALS.role == 'so-searchnode' %}
 {% set NODE_CONTAINERS = [
     'so-elasticsearch',
     'so-logstash',
     'so-nginx'
 ] %}
-{% endif %}

-{% if GLOBALS.role == 'so-heavynode' %}
+{% elif GLOBALS.role == 'so-heavynode' %}
 {% set NODE_CONTAINERS = [
     'so-curator',
     'so-elasticsearch',
@@ -71,9 +68,8 @@
     'so-strelka-manager',
     'so-strelka-filestream'
 ] %}
-{% endif %}

-{% if GLOBALS.role == 'so-import' %}
+{% elif GLOBALS.role == 'so-import' %}
 {% set NODE_CONTAINERS = [
     'so-dockerregistry',
     'so-elasticsearch',
@@ -85,17 +81,22 @@
     'so-nginx',
     'so-soc'
 ] %}
-{% endif %}

-{% if GLOBALS.role == 'so-receiver' %}
+{% elif GLOBALS.role == 'so-receiver' %}
 {% set NODE_CONTAINERS = [
     'so-logstash',
     'so-redis',
 ] %}
-{% endif %}

-{% if GLOBALS.role == 'so-idh' %}
+{% elif GLOBALS.role == 'so-idh' %}
 {% set NODE_CONTAINERS = [
     'so-idh',
 ] %}
+
+{% elif GLOBALS.role == 'so-sensor' %}
+{% set NODE_CONTAINERS = [] %}
+
+{% else %}
+{% set NODE_CONTAINERS = [] %}
+
 {% endif %}
--- a/salt/firewall/init.sls
+++ b/salt/firewall/init.sls
@@ -1,11 +1,6 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}

-disable_firewalld:
-  service.dead:
-    - name: firewalld
-    - enable: False
-
 create_sysconfig_iptables:
  file.touch:
    - name: /etc/sysconfig/iptables
@@ -18,10 +13,25 @@ iptables_config:
    - source: salt://firewall/iptables.jinja
    - template: jinja

+disable_firewalld:
+  service.dead:
+    - name: firewalld
+    - enable: False
+    - require:
+      - file: iptables_config
+
 iptables_restore:
  cmd.run:
    - name: iptables-restore < /etc/sysconfig/iptables

+enable_firewalld:
+  service.running:
+    - name: firewalld
+    - enable: True
+    - onfail:
+      - file: iptables_config
+      - cmd: iptables_restore
+
 {% else %}

 {{sls}}_state_not_allowed:
--- a/salt/firewall/ports/ports.yaml
+++ b/salt/firewall/ports/ports.yaml
@@ -32,6 +32,7 @@ firewall:
    elastic_agent_data:
      tcp:
        - 5055
+        - 9200
    endgame:
      tcp:
        - 3765
--- a/salt/influxdb/init.sls
+++ b/salt/influxdb/init.sls
@@ -117,6 +117,12 @@ influxdb-setup:
      - file: influxdb_curl_config
      - docker_container: so-influxdb

+metrics_link_file:
+  cmd.run:
+    - name: so-influxdb-manage dashboardpath "Security Onion Performance" > /opt/so/saltstack/local/salt/influxdb/metrics_link.txt
+    - require:
+      - docker_container: so-influxdb
+
 # Install cron job to determine size of influxdb for telegraf
 get_influxdb_size:
  cron.present:
--- a/salt/influxdb/metrics_link.txt
+++ b/salt/influxdb/metrics_link.txt
--- a/salt/manager/files/acng/acng.conf
+++ b/salt/manager/files/acng/acng.conf
@@ -1,96 +0,0 @@
-# This is a configuration file for apt-cacher-ng, a smart caching proxy for
-CacheDir: /var/cache/apt-cacher-ng
-LogDir: /var/log/apt-cacher-ng
-Port: 3142
-# BindAddress: localhost 192.168.7.254 publicNameOnMainInterface
-Remap-debrep: file:deb_mirror*.gz /debian ; file:backends_debian # Debian Archives
-Remap-uburep: file:ubuntu_mirrors /ubuntu ; file:backends_ubuntu.us # Ubuntu Archives
-Remap-cygwin: file:cygwin_mirrors /cygwin # ; file:backends_cygwin # incomplete, please create this file or specify preferred mirrors here
-Remap-alxrep: file:archlx_mirrors /archlinux # ; file:backend_archlx # Arch Linux
-Remap-centosmirrorlist: mirrorlist.centos.org
-Remap-centos: file:centos_mirrors ; file:backends_centos.us # Fedora Linux
-Remap-fedora: file:fedora_mirrors ; file:backends_fedora.us # Fedora Linux
-Remap-epel:   file:epel_mirrors ; file:backends_epel.us # Fedora EPEL
-Remap-slrep:  file:sl_mirrors # Scientific Linux
-Remap-gentoo: file:gentoo_mirrors.gz /gentoo ; file:backends_gentoo # Gentoo Archives
-#Remap-alpine: file:alpine_mirrors /alpine #; dl-cdn.alpinelinux.org # Alpine Archives
-Remap-alpine: dl-cdn.alpinelinux.org
-Remap-yarn:   registry.yarnpkg.com
-Remap-npm:    registry.npmjs.org
-Remap-node:   nodejs.org
-Remap-apache: file:apache_mirrors ; file:backends_apache.us
-Remap-salt:   repo.saltstack.com; https://repo.saltstack.com
-Remap-securityonion: http://repocache.securityonion.net ; file:securityonion
-# Remap-secdeb: security.debian.org
-ReportPage: acng-report.html
-# SocketPath:/var/run/apt-cacher-ng/socket
-UnbufferLogs: 1
-VerboseLog: 1
-ForeGround: 1
-# PidFile: /var/run/apt-cacher-ng/pid
-# Offlinemode: 0
-# ForceManaged: 0
-ExTreshold: 8
-# ExAbortOnProblems: 1
-# ExSuppressAdminNotification: 1
-# StupidFs: 0
-# ForwardBtsSoap: 1
-# DnsCacheSeconds: 1800
-# MaxStandbyConThreads: 8
-MaxConThreads: 120
-#
-# - static data that doesn't change silently ont he server (PFilePattern)
-# - volatile data that can be changed like every hour (VFilePattern)
-# - special static data that shared some file names with volatile data,
-#   and in doubt should be identified as static (SPfilePattern)
-# - a "whitelist pattern" with hints for the regular expiration job telling
-#   to keep the files even if they are not referenced by others, like crypto
-#   signatures with which clients begin their downloads (WfilePattern)
-#
-VfilePatternEx: (metalink\?repo=[0-9a-zA-Z-]+&arch=[0-9a-zA-Z_-]+|/\?release=[0-9]+&arch=|repodata/.*\.(xml|sqlite)\.(gz|bz2)|APKINDEX.tar.gz|filelists\.xml\.gz|filelists\.sqlite\.bz2|repomd\.xml|packages\.[a-zA-Z][a-zA-Z]\.gz)
-PfilePatternEx: (/dists/.*/by-hash/.*|\.tgz|\.tar|\.xz|\.bz2|\.rpm|\.apk)$
-# WfilePatternEx:
-# SPfilePatternEx:
-
-Debug:1
-# ExposeOrigin: 0
-# LogSubmittedOrigin: 0
-# UserAgent: Yet Another HTTP Client/1.2.3p4
-# RecompBz2: 0
-# NetworkTimeout: 60
-
-# DontCacheRequested: linux-.*_10\...\.Custo._i386
-# DontCacheRequested: 192.168.0 ^10\..* 172.30
-# DontCacheResolved: ubuntumirror.local.net
-DontCache: mirrorlist.centos.org
-
-# DirPerms: 00755
-# FilePerms: 00664
-
-LocalDirs: acng-doc /usr/share/doc/apt-cacher-ng
-# PrecacheFor: debrep/dists/unstable/*/source/Sources* debrep/dists/unstable/*/binary-amd64/Packages*
-# RequestAppendix: X-Tracking-Choice: do-not-track\r\n
-# ConnectProto: v6 v4
-# KeepExtraVersions: 0
-# UseWrap: 0
-FreshIndexMaxAge: 300
-# AllowUserPorts: 80
-RedirMax: 6
-# VfileUseRangeOps is set for fedora volatile files on mirrors that dont to range
-VfileUseRangeOps: -1
-# PassThroughPattern: private-ppa\.launchpad\.net:443$
-# PassThroughPattern: .* # this would allow CONNECT to everything
-PassThroughPattern: (repo\.securityonion\.net:443|download\.docker\.com:443|mirrors\.fedoraproject\.org:443|packages\.wazuh\.com:443|repo\.saltstack\.com:443|repo\.saltproject\.io:443|yum\.dockerproject\.org:443|download\.docker\.com:443|registry\.npmjs\.org:443|registry\.yarnpkg\.com:443)$ # yarn/npm pkg, cant to http :/
-# ResponseFreezeDetectTime: 500
-# ReuseConnections: 1
-# PipelineDepth: 255
-# CApath: /etc/ssl/certs
-# CAfile:
-# OptProxyTimeout: -1
-# MaxDlSpeed: 500
-# MaxInresponsiveDlSize: 64000
-# BadRedirDetectMime: text/html
-{% set proxy = salt['pillar.get']('manager:proxy') -%}
-{% if proxy -%}
-Proxy: {{ proxy }}
-{% endif -%}
--- a/salt/mysql/init.sls
+++ b/salt/mysql/init.sls
@@ -106,22 +106,6 @@ so-mysql:
    - require:
      - file: mysqlcnf
      - file: mysqlpass
-  cmd.run:
-    - name: until nc -z {{ GLOBALS.manager }} 3306; do sleep 1; done
-    - timeout: 600
-    - onchanges:
-      - docker_container: so-mysql
-  module.run:
-    - so.mysql_conn:
-      - retry: 300
-    - onchanges:
-      - cmd: so-mysql
-
-append_so-mysql_so-status.conf:
-  file.append:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - text: so-mysql
-
 {% endif %}

 {% else %}
--- a/salt/soc/defaults.map.jinja
+++ b/salt/soc/defaults.map.jinja
@@ -1,8 +1,8 @@
 {% import_yaml 'soc/defaults.yaml' as SOCDEFAULTS %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from 'docker/docker.map.jinja' import DOCKER -%}
-{%- set INFLUXDB_TOKEN = salt['pillar.get']('secrets:influx_token') %}
-{%- set METRICS_LINK = salt['cmd.run']('so-influxdb-manage dashboardpath "Security Onion Performance"') %}
+{% set INFLUXDB_TOKEN = salt['pillar.get']('secrets:influx_token') %}
+{% import_text 'influxdb/metrics_link.txt' as METRICS_LINK %}

 {% for module, application_url in GLOBALS.application_urls.items() %}
 {%   do SOCDEFAULTS.soc.server.modules[module].update({'hostUrl': application_url}) %}
@@ -20,7 +20,7 @@
 {% do SOCDEFAULTS.soc.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %}
 {% do SOCDEFAULTS.soc.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %}
 {% for tool in SOCDEFAULTS.soc.server.client.tools %}
-{%   if tool.name == "toolInfluxDb" %}
+{%   if tool.name == "toolInfluxDb" and METRICS_LINK | length > 0 %}
 {%     do tool.update({'link': METRICS_LINK}) %}
 {%   endif %}
 {% endfor %}
--- a/salt/top.sls
+++ b/salt/top.sls
@@ -28,10 +28,6 @@ base:
    - salt.minion-state-apply-test
    - salt.minion

-  'G@os:Rocky and G@saltversion:{{saltversion}}':
-    - match: compound
-    - yum.packages
-
  '* and G@saltversion:{{saltversion}}':
    - match: compound
    - salt.minion
@@ -63,6 +59,7 @@ base:
    {%- endif %}
    - schedule
    - docker_clean
+    - elastic-fleet.install_agent_grid

  '*_eval and G@saltversion:{{saltversion}}':
    - match: compound
@@ -109,6 +106,7 @@ base:
    - playbook
    - redis
    {%- endif %}
+    - elastic-fleet.install_agent_grid
    - docker_clean

  '*_manager and G@saltversion:{{saltversion}}':
@@ -150,6 +148,7 @@ base:
    - schedule
    - soctopus
    - playbook
+    - elastic-fleet.install_agent_grid
    - docker_clean

  '*_standalone and G@saltversion:{{saltversion}}':
@@ -202,6 +201,7 @@ base:
    - playbook
    - docker_clean
    - elastic-fleet
+    - elastic-fleet.install_agent_grid

  '*_searchnode and G@saltversion:{{saltversion}}':
    - match: compound
@@ -217,6 +217,7 @@ base:
    - logstash
    {%- endif %}
    - schedule
+    - elastic-fleet.install_agent_grid
    - docker_clean

  '*_managersearch and G@saltversion:{{saltversion}}':
@@ -258,6 +259,7 @@ base:
    - schedule
    - soctopus
    - playbook
+    - elastic-fleet.install_agent_grid
    - docker_clean

  '*_heavynode and G@saltversion:{{saltversion}}':
@@ -286,6 +288,7 @@ base:
    - zeek
    {%- endif %}
    - schedule
+    - elastic-fleet.install_agent_grid
    - docker_clean
  
  '*_import and G@saltversion:{{saltversion}}':
@@ -317,6 +320,7 @@ base:
    - zeek
    - schedule
    - docker_clean
+    - elastic-fleet.install_agent_grid
    - elastic-fleet

  '*_receiver and G@saltversion:{{saltversion}}':
@@ -332,6 +336,7 @@ base:
    - redis
    {%- endif %}
    - schedule
+    - elastic-fleet.install_agent_grid
    - docker_clean

  '*_idh and G@saltversion:{{saltversion}}':
@@ -341,6 +346,7 @@ base:
    - telegraf
    - firewall
    - schedule
+    - elastic-fleet.install_agent_grid
    - docker_clean
    - idh

--- a/salt/yum/packages.sls
+++ b/salt/yum/packages.sls
@@ -1,4 +0,0 @@
-install_yum_utils:
-  pkg.installed:
-    - name: yum-utils
-
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1832,6 +1832,7 @@ reinstall_init() {
 		# Backup (and erase) directories in /nsm to prevent app errors
 		backup_dir /nsm/mysql "$date_string"
 		backup_dir /nsm/kratos "$date_string"
+		backup_dir /nsm/influxdb "$date_string"

 		# Remove the old launcher package in case the config changes
 		remove_package launcher-final
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -611,7 +611,8 @@ if ! [[ -f $install_opt_file ]]; then
 		title "Seeding the docker registry"
 		docker_seed_registry
 		title "Applying the manager state"
-		logCmd "salt-call state.apply -l info manager" 
+		logCmd "salt-call state.apply -l info manager"
+		logCmd "salt-call state.apply influxdb -l info"
 		logCmd "salt-call state.highstate -l info"
 		add_web_user
 		info "Restarting SOC to pick up initial user"
--- a/setup/so-verify
+++ b/setup/so-verify
@@ -36,7 +36,7 @@ log_has_errors() {
        grep -vE "The Salt Master has cached the public key for this node" | \
        grep -vE "Minion failed to authenticate with the master" | \
        grep -vE "Failed to connect to ::1" | \
-	grep -vE "Failed to set locale" | \
+        grep -vE "Failed to set locale" | \
        grep -vE "perl-Error-" | \
        grep -vE "Failed:\s*?[0-9]+" | \
        grep -vE "Status .* was not found" | \