Support sigma playbooks for airgap

update vm state applid for pillar_push_map

salt/common/tools/sbin/so-log-check

@@ -231,7 +231,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint|armis|o365_metrics|microsoft_sentinel|snyk|cyera|island_browser).*user so_kibana lacks the required permissions \[(logs|metrics)-\1" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user. This error should not be seen on fresh ES 9.3.3 installs or after SO 3.1.0 with soups addition of check_transform_health_and_reauthorize()
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
 fi
 

salt/common/tools/sbin/so-restart

@@ -35,9 +35,6 @@ case $1 in
     "elastic-fleet"|"elasticfleet")
         docker_check_running "elastic-fleet" "--stop"
         docker rm "so-elastic-fleet" 2> /dev/null
-        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
-        rm -rf /opt/so/conf/elastic-fleet/state
-
         salt-call state.apply elasticfleet queue=True
         ;;
     *)

salt/common/tools/sbin/so-stop

@@ -29,8 +29,6 @@ case $1 in
 	"elasticfleet"|"elastic-fleet")
         docker_check_running "elastic-fleet" "--stop"
         docker rm "so-elastic-fleet" 2> /dev/null
-        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
-        rm -rf /opt/so/conf/elastic-fleet/state
         ;;
     *)
         docker_check_running "$1" "--stop"

salt/elasticfleet/enabled.sls

@@ -11,10 +11,6 @@
 
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
-{#   Prevent Elastic Agent from re-enrolling with a new agent.id everytime the container starts up.
-       - if a fresh enrollment is needed use 'so-stop elasticfleet'
-       #}
-{%   set ENROLLED = salt['file.file_exists']('/opt/so/conf/elastic-fleet/state/fleet.enc') %}
 
 include:
   - ca
@@ -70,7 +66,6 @@ so-elastic-fleet:
       - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro
       - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
-      - /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state
       - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs 
      {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
         {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
@@ -78,7 +73,6 @@ so-elastic-fleet:
         {% endfor %}
       {% endif %}      
     - environment:
-      {% if not ENROLLED %}
       - FLEET_SERVER_ENABLE=true
       - FLEET_URL=https://{{ GLOBALS.hostname }}:8220
       - FLEET_SERVER_ELASTICSEARCH_HOST=https://{{ GLOBALS.manager }}:9200
@@ -88,9 +82,6 @@ so-elastic-fleet:
       - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key
       - FLEET_CA=/etc/pki/tls/certs/intca.crt     
       - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
-      {% endif %}
-      - STATE_PATH=/usr/share/elastic-agent/state
-      - CONFIG_PATH=/usr/share/elastic-agent/state
       - LOGS_PATH=logs
       {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
         {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
@@ -109,7 +100,6 @@ so-elastic-fleet:
       - x509: etc_elasticfleet_crt
     - require:
       - file: trusttheca
-      - file: eastatedir
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
 

salt/manager/tools/sbin/soup

@@ -980,6 +980,10 @@ up_to_3.2.0() {
 
   pin_elasticsearch_data_retention_method
 
+  # Run so-elastic-fleet-es-url update with --force to ensure eval/import have
+  #  configured so-manager_elasicsearch as the default output for both monitoring and logs
+  /usr/sbin/so-elastic-fleet-es-url-update --force
+
   INSTALLEDVERSION=3.2.0
 }
 

salt/repo/client/oracle.sls

@@ -60,6 +60,9 @@ so_repo:
   {% endif %}
     - enabled: 1
     - gpgcheck: 1
+  {% if not GLOBALS.is_manager %}
+    - sslverify: 0
+  {% endif %}
 
 # Only assign the kernel repo once this node's running salt matches the version this
 # SO release ships. During a soup the grid is mid-salt-upgrade; gating here keeps the
@@ -77,6 +80,9 @@ so_kernel_repo:
   {% endif %}
     - enabled: 1
     - gpgcheck: 1
+  {% if not GLOBALS.is_manager %}
+    - sslverify: 0
+  {% endif %}
     # Supplementary kernel repo: tolerate it being empty/unreachable (e.g. before the
     # manager has populated /nsm/kernelrepo) so a missing repomd.xml can't make every
     # dnf/pkg operation on the grid fail.

salt/soc/defaults.yaml

@@ -1512,6 +1512,10 @@ soc:
                 rulesetName: sos-resources-ag
                 branch: main
                 folder: securityonion-normalized
+              - repo: file:///nsm/airgap-resources/playbooks/securityonion-resources-playbooks
+                rulesetName: sos-published-ag
+                branch: published
+                folder: sigma
         assistant:
           systemPromptAddendum: ""
           systemPromptAddendumMaxLength: 50000

salt/soc/files/soc/sigma_playbook_pipeline.yaml

@@ -12,6 +12,30 @@ transformations:
         process.command_line: process.command_line.caseless
         process.parent.command_line: process.parent.command_line.caseless
         file.path: file.path.caseless
+    # entity_id pivots must also match processes that were already running when the
+    # agent started: Defend emits already_running (event.type:info), not start.
+    # Kept out of Playbook sigma query because Sysmon has no equivalent concept.
+    # contains_field is exact, so child pivots (process.parent.entity_id) stay
+    # start-only. Drop must precede add;
+    - id: playbook_process_lifecycle_drop_start_scope
+      type: drop_detection_item
+      field_name_conditions:
+        - type: include_fields
+          fields: ['event.type']
+      rule_conditions:
+        - type: logsource
+          category: process_creation
+        - type: contains_field
+          field: process.entity_id
+    - id: playbook_process_lifecycle_add-fields
+      type: add_condition
+      conditions:
+        event.type: ['start', 'info']
+      rule_conditions:
+        - type: logsource
+          category: process_creation
+        - type: contains_field
+          field: process.entity_id
     # file_activity: playbook-only pseudo-category spanning all file operations.
     - id: playbook_file_activity_add-fields
       type: add_condition

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -68,13 +68,25 @@ transformations:
       - type: logsource
         category: antivirus
     # OS-agnostic process_creation scoping for product-less (NIDS/host-pivot) rules.
+    # pySigma: rule_cond_expr requires rule_conditions as a mapping, not a list.
     - id: process_creation_os_agnostic
       type: add_condition
       conditions:
         event.category: process
       rule_conditions:
-      - type: logsource
-        category: process_creation
+        pc_cat:
+          type: logsource
+          category: process_creation
+        pc_win:
+          type: logsource
+          product: windows
+        pc_mac:
+          type: logsource
+          product: macos
+        pc_lin:
+          type: logsource
+          product: linux
+      rule_cond_expr: "pc_cat and not (pc_win or pc_mac or pc_lin)"
     # Transforms the `Hashes` field to ECS fields
     # ECS fields are used by the hash fields emitted by Elastic Defend
     # If shipped with Elastic Agent, sysmon logs will also have hashes mapped to ECS fields    
@@ -630,4 +642,4 @@ transformations:
         tags: '*file'
       rule_conditions:
       - type: logsource
-        category: file
+        category: file