Merge pull request #16113 from Security-Onion-Solutions/reyesj2-pach-1

run es-url-updae prior to upgrade

salt/common/tools/sbin/so-restart

@@ -35,9 +35,6 @@ case $1 in
     "elastic-fleet"|"elasticfleet")
         docker_check_running "elastic-fleet" "--stop"
         docker rm "so-elastic-fleet" 2> /dev/null
-        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
-        rm -rf /opt/so/conf/elastic-fleet/state
-
         salt-call state.apply elasticfleet queue=True
         ;;
     *)

salt/common/tools/sbin/so-stop

@@ -29,8 +29,6 @@ case $1 in
 	"elasticfleet"|"elastic-fleet")
         docker_check_running "elastic-fleet" "--stop"
         docker rm "so-elastic-fleet" 2> /dev/null
-        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
-        rm -rf /opt/so/conf/elastic-fleet/state
         ;;
     *)
         docker_check_running "$1" "--stop"

salt/common/tools/sbin_jinja/so-salt-minion-check

@@ -25,7 +25,7 @@ LAST_HIGHSTATE_END=$([ -e "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/
 LAST_HEALTHCHECK_STATE_APPLY=$([ -e "/opt/so/log/salt/state-apply-test" ] && date -r /opt/so/log/salt/state-apply-test +%s || echo 0)
 # SETTING THRESHOLD TO ANYTHING UNDER 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
 # THRESHOLD is derived from the salt schedule highstate interval + 1 hour, so the minion-check grace period tracks the schedule automatically.
-THRESHOLD=$(( ({{ SCHEDULEMERGED.highstate_interval_hours }} + 1) * 3600 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
+THRESHOLD=$(( ({{ SCHEDULEMERGED.highstate_interval_minutes }} + 60) * 60 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
 THRESHOLD_DATE=$((LAST_HEALTHCHECK_STATE_APPLY+THRESHOLD))
 
 logCmd() {

salt/elasticfleet/enabled.sls

@@ -11,10 +11,6 @@
 
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
-{#   Prevent Elastic Agent from re-enrolling with a new agent.id everytime the container starts up.
-       - if a fresh enrollment is needed use 'so-stop elasticfleet'
-       #}
-{%   set ENROLLED = salt['file.file_exists']('/opt/so/conf/elastic-fleet/state/fleet.enc') %}
 
 include:
   - ca
@@ -70,7 +66,6 @@ so-elastic-fleet:
       - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro
       - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
-      - /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state
       - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs 
      {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
         {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
@@ -78,7 +73,6 @@ so-elastic-fleet:
         {% endfor %}
       {% endif %}      
     - environment:
-      {% if not ENROLLED %}
       - FLEET_SERVER_ENABLE=true
       - FLEET_URL=https://{{ GLOBALS.hostname }}:8220
       - FLEET_SERVER_ELASTICSEARCH_HOST=https://{{ GLOBALS.manager }}:9200
@@ -88,9 +82,6 @@ so-elastic-fleet:
       - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key
       - FLEET_CA=/etc/pki/tls/certs/intca.crt     
       - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
-      {% endif %}
-      - STATE_PATH=/usr/share/elastic-agent/state
-      - CONFIG_PATH=/usr/share/elastic-agent/state
       - LOGS_PATH=logs
       {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
         {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
@@ -109,7 +100,6 @@ so-elastic-fleet:
       - x509: etc_elasticfleet_crt
     - require:
       - file: trusttheca
-      - file: eastatedir
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
 

salt/elasticsearch/defaults.yaml

@@ -1,5 +1,6 @@
 elasticsearch:
   enabled: false
+  esheap: '600m'
   version: 9.3.7
   index_clean: true
   data_retention_method: DLM

salt/manager/tools/sbin/soup

@@ -442,7 +442,13 @@ get_soup_script_hashes() {
 }
 
 highstate() {
-  # Run a highstate.
+  # Run a highstate with a retry attempt.
+  if salt-call state.highstate -l info queue=True; then
+    return 0
+  fi
+
+  echo "Initial highstate attempt had a problem; retrying in 30 seconds."
+  sleep 30
   salt-call state.highstate -l info queue=True
 }
 
@@ -974,6 +980,10 @@ up_to_3.2.0() {
 
   pin_elasticsearch_data_retention_method
 
+  # Run so-elastic-fleet-es-url update with --force to ensure eval/import have
+  #  configured so-manager_elasicsearch as the default output for both monitoring and logs
+  /usr/sbin/so-elastic-fleet-es-url-update --force
+
   INSTALLEDVERSION=3.2.0
 }
 

salt/salt/defaults.yaml

@@ -6,4 +6,4 @@ salt:
     batch: '25%'
     batch_wait: 15
   schedule:
-    highstate_interval_hours: 2
+    highstate_interval_minutes: 120

salt/salt/highstate_schedule.sls

@@ -1,11 +1,15 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from 'salt/schedule.map.jinja' import SCHEDULEMERGED %}
 
+{# splay a quarter of the interval, clamped to [5 min, 30 min], so short intervals
+   don't get jitter larger than the interval itself #}
+{% set SPLAY = [[(SCHEDULEMERGED.highstate_interval_minutes * 60 // 4) | int, 300] | max, 1800] | min %}
+
 highstate_schedule:
   schedule.present:
     - function: state.highstate
-    - hours: {{ SCHEDULEMERGED.highstate_interval_hours }}
+    - minutes: {{ SCHEDULEMERGED.highstate_interval_minutes }}
     - maxrunning: 1
 {% if not GLOBALS.is_manager %}
-    - splay: 1800
+    - splay: {{ SPLAY }}
 {% endif %}

salt/salt/soc_salt.yaml

@@ -31,9 +31,11 @@ salt:
       global: True
       advanced: True
   schedule:
-    highstate_interval_hours:
-      description: How often every minion in the grid runs a scheduled state.highstate, in hours. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. The salt-minion health check restarts a minion if its last highstate is older than this value plus one hour.
+    highstate_interval_minutes:
+      description: How often every minion in the grid runs a scheduled state.highstate, in minutes. Minimum 15 minutes. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. If Auto Apply is disabled, set this to the 15-minute minimum so changes are still picked up promptly. The salt-minion health check restarts a minion if its last state apply is older than this value plus one hour.
       forcedType: int
       helpLink: push
       global: True
       advanced: True
+      regex: '^(1[5-9]|[2-9][0-9]|[1-9][0-9]{2,4})$'
+      regexFailureMessage: The value must be an integer of at least 15 minutes (maximum 99999).

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -46,20 +46,24 @@ transformations:
       - type: logsource
         product: opencanary
     # Maps "antivirus" category to Windows Defender logs shipped by Elastic Agent Winlog Integration
+    # and to Elastic Defend malware alerts
     # winlog.event_data.threat_name has to be renamed prior to ingestion, it is originally winlog.event_data.Threat Name
-    - id: antivirus_field-mappings_windows-defender
+    - id: antivirus_field-mappings
       type: field_name_mapping
       mapping:
-        Signature: winlog.event_data.threat_name
+        Signature:
+          - winlog.event_data.threat_name
+          - rule.name
       rule_conditions:
       - type: logsource
         category: antivirus
-    - id: antivirus_add-fields_windows-defender
+    - id: antivirus_add-fields
       type: add_condition
       conditions:
-        winlog.channel: 'Microsoft-Windows-Windows Defender/Operational'
-        winlog.provider_name: 'Microsoft-Windows-Windows Defender'
-        event.code: "1116"
+        event.code:
+          - "1116"
+          - "malicious_file"
+          - "memory_signature"
       rule_conditions:
       - type: logsource
         category: antivirus

salt/vars/elasticsearch.map.jinja

@@ -1,10 +1,11 @@
 {% import 'vars/init.map.jinja' as INIT %}
+{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 
 {%
   set ELASTICSEARCH_GLOBALS = {
     'elasticsearch': {
       'es_cluster_name': INIT.PILLAR.elasticsearch.config.cluster.name,
-      'es_heap': INIT.PILLAR.elasticsearch.esheap
+      'es_heap': INIT.PILLAR.elasticsearch.get('esheap', ELASTICSEARCHDEFAULTS.elasticsearch.esheap)
     }
   }
 %}