Support sigma playbooks for airgap

update vm state applid for pillar_push_map

salt/common/tools/sbin/so-log-check

@@ -231,7 +231,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint|armis|o365_metrics|microsoft_sentinel|snyk|cyera|island_browser).*user so_kibana lacks the required permissions \[(logs|metrics)-\1" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user. This error should not be seen on fresh ES 9.3.3 installs or after SO 3.1.0 with soups addition of check_transform_health_and_reauthorize()
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
 fi
 

salt/common/tools/sbin/so-restart

@@ -35,9 +35,6 @@ case $1 in
     "elastic-fleet"|"elasticfleet")
         docker_check_running "elastic-fleet" "--stop"
         docker rm "so-elastic-fleet" 2> /dev/null
-        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
-        rm -rf /opt/so/conf/elastic-fleet/state
-
         salt-call state.apply elasticfleet queue=True
         ;;
     *)

salt/common/tools/sbin/so-stop

@@ -29,8 +29,6 @@ case $1 in
 	"elasticfleet"|"elastic-fleet")
         docker_check_running "elastic-fleet" "--stop"
         docker rm "so-elastic-fleet" 2> /dev/null
-        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
-        rm -rf /opt/so/conf/elastic-fleet/state
         ;;
     *)
         docker_check_running "$1" "--stop"

salt/common/tools/sbin_jinja/so-salt-minion-check

@@ -25,7 +25,7 @@ LAST_HIGHSTATE_END=$([ -e "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/
 LAST_HEALTHCHECK_STATE_APPLY=$([ -e "/opt/so/log/salt/state-apply-test" ] && date -r /opt/so/log/salt/state-apply-test +%s || echo 0)
 # SETTING THRESHOLD TO ANYTHING UNDER 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
 # THRESHOLD is derived from the salt schedule highstate interval + 1 hour, so the minion-check grace period tracks the schedule automatically.
-THRESHOLD=$(( ({{ SCHEDULEMERGED.highstate_interval_hours }} + 1) * 3600 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
+THRESHOLD=$(( ({{ SCHEDULEMERGED.highstate_interval_minutes }} + 60) * 60 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
 THRESHOLD_DATE=$((LAST_HEALTHCHECK_STATE_APPLY+THRESHOLD))
 
 logCmd() {

salt/elasticfleet/enabled.sls

@@ -11,10 +11,6 @@
 
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
-{#   Prevent Elastic Agent from re-enrolling with a new agent.id everytime the container starts up.
-       - if a fresh enrollment is needed use 'so-stop elasticfleet'
-       #}
-{%   set ENROLLED = salt['file.file_exists']('/opt/so/conf/elastic-fleet/state/fleet.enc') %}
 
 include:
   - ca
@@ -70,7 +66,6 @@ so-elastic-fleet:
       - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro
       - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
-      - /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state
       - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs 
      {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
         {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
@@ -78,7 +73,6 @@ so-elastic-fleet:
         {% endfor %}
       {% endif %}      
     - environment:
-      {% if not ENROLLED %}
       - FLEET_SERVER_ENABLE=true
       - FLEET_URL=https://{{ GLOBALS.hostname }}:8220
       - FLEET_SERVER_ELASTICSEARCH_HOST=https://{{ GLOBALS.manager }}:9200
@@ -88,9 +82,6 @@ so-elastic-fleet:
       - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key
       - FLEET_CA=/etc/pki/tls/certs/intca.crt     
       - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
-      {% endif %}
-      - STATE_PATH=/usr/share/elastic-agent/state
-      - CONFIG_PATH=/usr/share/elastic-agent/state
       - LOGS_PATH=logs
       {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
         {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
@@ -109,7 +100,6 @@ so-elastic-fleet:
       - x509: etc_elasticfleet_crt
     - require:
       - file: trusttheca
-      - file: eastatedir
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
 

salt/elasticsearch/defaults.yaml

@@ -1,5 +1,6 @@
 elasticsearch:
   enabled: false
+  esheap: '600m'
   version: 9.3.7
   index_clean: true
   data_retention_method: DLM

salt/manager/tools/sbin/soup

@@ -442,7 +442,13 @@ get_soup_script_hashes() {
 }
 
 highstate() {
-  # Run a highstate.
+  # Run a highstate with a retry attempt.
+  if salt-call state.highstate -l info queue=True; then
+    return 0
+  fi
+
+  echo "Initial highstate attempt had a problem; retrying in 30 seconds."
+  sleep 30
   salt-call state.highstate -l info queue=True
 }
 
@@ -974,6 +980,10 @@ up_to_3.2.0() {
 
   pin_elasticsearch_data_retention_method
 
+  # Run so-elastic-fleet-es-url update with --force to ensure eval/import have
+  #  configured so-manager_elasicsearch as the default output for both monitoring and logs
+  /usr/sbin/so-elastic-fleet-es-url-update --force
+
   INSTALLEDVERSION=3.2.0
 }
 

salt/reactor/pillar_push_map.yaml

@@ -242,7 +242,7 @@ versionlock:
 # grain (compound supports nested grain matching via G@<key>:<subkey>:<value>).
 # pillar/vm/soc_vm.sls write path is referenced at salt/_runners/setup_hypervisor.py:856.
 vm:
-  - state: vm
+  - state: vm.user
     tgt: 'G@salt-cloud:driver:libvirt'
 
 # zeek: sensor_roles + so-import (5 roles).

salt/repo/client/oracle.sls

@@ -60,6 +60,9 @@ so_repo:
   {% endif %}
     - enabled: 1
     - gpgcheck: 1
+  {% if not GLOBALS.is_manager %}
+    - sslverify: 0
+  {% endif %}
 
 # Only assign the kernel repo once this node's running salt matches the version this
 # SO release ships. During a soup the grid is mid-salt-upgrade; gating here keeps the
@@ -77,6 +80,9 @@ so_kernel_repo:
   {% endif %}
     - enabled: 1
     - gpgcheck: 1
+  {% if not GLOBALS.is_manager %}
+    - sslverify: 0
+  {% endif %}
     # Supplementary kernel repo: tolerate it being empty/unreachable (e.g. before the
     # manager has populated /nsm/kernelrepo) so a missing repomd.xml can't make every
     # dnf/pkg operation on the grid fail.

salt/salt/defaults.yaml

@@ -6,4 +6,4 @@ salt:
     batch: '25%'
     batch_wait: 15
   schedule:
-    highstate_interval_hours: 2
+    highstate_interval_minutes: 120

salt/salt/highstate_schedule.sls

@@ -1,11 +1,15 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from 'salt/schedule.map.jinja' import SCHEDULEMERGED %}
 
+{# splay a quarter of the interval, clamped to [5 min, 30 min], so short intervals
+   don't get jitter larger than the interval itself #}
+{% set SPLAY = [[(SCHEDULEMERGED.highstate_interval_minutes * 60 // 4) | int, 300] | max, 1800] | min %}
+
 highstate_schedule:
   schedule.present:
     - function: state.highstate
-    - hours: {{ SCHEDULEMERGED.highstate_interval_hours }}
+    - minutes: {{ SCHEDULEMERGED.highstate_interval_minutes }}
     - maxrunning: 1
 {% if not GLOBALS.is_manager %}
-    - splay: 1800
+    - splay: {{ SPLAY }}
 {% endif %}

salt/salt/soc_salt.yaml

@@ -31,9 +31,11 @@ salt:
       global: True
       advanced: True
   schedule:
-    highstate_interval_hours:
+    highstate_interval_minutes:
-      description: How often every minion in the grid runs a scheduled state.highstate, in hours. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. The salt-minion health check restarts a minion if its last highstate is older than this value plus one hour.
+      description: How often every minion in the grid runs a scheduled state.highstate, in minutes. Minimum 15 minutes. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. If Auto Apply is disabled, set this to the 15-minute minimum so changes are still picked up promptly. The salt-minion health check restarts a minion if its last state apply is older than this value plus one hour.
       forcedType: int
       helpLink: push
       global: True
       advanced: True
+      regex: '^(1[5-9]|[2-9][0-9]|[1-9][0-9]{2,4})$'
+      regexFailureMessage: The value must be an integer of at least 15 minutes (maximum 99999).

salt/soc/defaults.yaml

@@ -1512,6 +1512,10 @@ soc:
                 rulesetName: sos-resources-ag
                 branch: main
                 folder: securityonion-normalized
+              - repo: file:///nsm/airgap-resources/playbooks/securityonion-resources-playbooks
+                rulesetName: sos-published-ag
+                branch: published
+                folder: sigma
         assistant:
           systemPromptAddendum: ""
           systemPromptAddendumMaxLength: 50000

salt/soc/files/soc/sigma_playbook_pipeline.yaml

@@ -12,6 +12,30 @@ transformations:
         process.command_line: process.command_line.caseless
         process.parent.command_line: process.parent.command_line.caseless
         file.path: file.path.caseless
+    # entity_id pivots must also match processes that were already running when the
+    # agent started: Defend emits already_running (event.type:info), not start.
+    # Kept out of Playbook sigma query because Sysmon has no equivalent concept.
+    # contains_field is exact, so child pivots (process.parent.entity_id) stay
+    # start-only. Drop must precede add;
+    - id: playbook_process_lifecycle_drop_start_scope
+      type: drop_detection_item
+      field_name_conditions:
+        - type: include_fields
+          fields: ['event.type']
+      rule_conditions:
+        - type: logsource
+          category: process_creation
+        - type: contains_field
+          field: process.entity_id
+    - id: playbook_process_lifecycle_add-fields
+      type: add_condition
+      conditions:
+        event.type: ['start', 'info']
+      rule_conditions:
+        - type: logsource
+          category: process_creation
+        - type: contains_field
+          field: process.entity_id
     # file_activity: playbook-only pseudo-category spanning all file operations.
     - id: playbook_file_activity_add-fields
       type: add_condition

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -46,31 +46,47 @@ transformations:
       - type: logsource
         product: opencanary
     # Maps "antivirus" category to Windows Defender logs shipped by Elastic Agent Winlog Integration
+    # and to Elastic Defend malware alerts
     # winlog.event_data.threat_name has to be renamed prior to ingestion, it is originally winlog.event_data.Threat Name
-    - id: antivirus_field-mappings_windows-defender
+    - id: antivirus_field-mappings
       type: field_name_mapping
       mapping:
-        Signature: winlog.event_data.threat_name
+        Signature:
+          - winlog.event_data.threat_name
+          - rule.name
       rule_conditions:
       - type: logsource
         category: antivirus
-    - id: antivirus_add-fields_windows-defender
+    - id: antivirus_add-fields
       type: add_condition
       conditions:
-        winlog.channel: 'Microsoft-Windows-Windows Defender/Operational'
+        event.code:
-        winlog.provider_name: 'Microsoft-Windows-Windows Defender'
+          - "1116"
-        event.code: "1116"
+          - "malicious_file"
+          - "memory_signature"
       rule_conditions:
       - type: logsource
         category: antivirus
     # OS-agnostic process_creation scoping for product-less (NIDS/host-pivot) rules.
+    # pySigma: rule_cond_expr requires rule_conditions as a mapping, not a list.
     - id: process_creation_os_agnostic
       type: add_condition
       conditions:
         event.category: process
       rule_conditions:
-      - type: logsource
+        pc_cat:
-        category: process_creation
+          type: logsource
+          category: process_creation
+        pc_win:
+          type: logsource
+          product: windows
+        pc_mac:
+          type: logsource
+          product: macos
+        pc_lin:
+          type: logsource
+          product: linux
+      rule_cond_expr: "pc_cat and not (pc_win or pc_mac or pc_lin)"
     # Transforms the `Hashes` field to ECS fields
     # ECS fields are used by the hash fields emitted by Elastic Defend
     # If shipped with Elastic Agent, sysmon logs will also have hashes mapped to ECS fields    

salt/vars/elasticsearch.map.jinja

@@ -1,10 +1,11 @@
 {% import 'vars/init.map.jinja' as INIT %}
+{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 
 {%
   set ELASTICSEARCH_GLOBALS = {
     'elasticsearch': {
       'es_cluster_name': INIT.PILLAR.elasticsearch.config.cluster.name,
-      'es_heap': INIT.PILLAR.elasticsearch.esheap
+      'es_heap': INIT.PILLAR.elasticsearch.get('esheap', ELASTICSEARCHDEFAULTS.elasticsearch.esheap)
     }
   }
 %}