Merge branch 'Security-Onion-Solutions:2.4/dev' into 2.4/dev

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json

@@ -27,6 +27,7 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }
 {%- endraw -%}

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json

@@ -28,6 +28,7 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }
 {%- endraw -%}

salt/elasticfleet/files/integrations-optional/sublime_platform.json

@@ -39,6 +39,7 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }
 {%- endraw -%}

salt/elasticfleet/files/integrations/endpoints-initial/osquery.json

@@ -16,5 +16,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json

@@ -73,5 +73,6 @@
     "system-system/metrics": {
       "enabled": false
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json

@@ -63,5 +63,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json

@@ -102,5 +102,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -28,5 +28,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json

@@ -16,5 +16,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json

@@ -72,5 +72,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json

@@ -28,5 +28,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json

@@ -29,5 +29,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json

@@ -36,5 +36,6 @@
     "system-system/metrics": {
       "enabled": false
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json

@@ -16,5 +16,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json

@@ -36,5 +36,6 @@
     "system-system/metrics": {
       "enabled": false
     }
-  }
+  },
+  "force": true
 }

salt/elasticsearch/enabled.sls

@@ -26,7 +26,12 @@ so-elasticsearch:
     - networks:
       - sobridge:
         - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }}
-    - extra_hosts:  {{ LOGSTASH_NODES }}
+    - extra_hosts:
+    {% for node in LOGSTASH_NODES %}
+    {%   for hostname, ip in node.items() %}
+      - {{hostname}}:{{ip}}
+    {%   endfor %}
+    {% endfor %}
     {% if DOCKER.containers['so-elasticsearch'].extra_hosts %}
       {% for XTRAHOST in DOCKER.containers['so-elasticsearch'].extra_hosts %}
       - {{ XTRAHOST }}

salt/logstash/enabled.sls

@@ -8,12 +8,7 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
 {%   from 'logstash/map.jinja' import LOGSTASH_MERGED %}
-{%   from 'logstash/map.jinja' import REDIS_NODES %}
-{#   we append the manager here so that it is added to extra_hosts so the heavynode can resolve it #}
-{#   we cannont append in the logstash/map.jinja because then it would be added to the 0900_input_redis.conf #}
-{%   if GLOBALS.role == 'so-heavynode' %}
-{%     do REDIS_NODES.append({GLOBALS.manager:GLOBALS.manager_ip}) %}
-{%   endif %}
+{%   from 'logstash/map.jinja' import LOGSTASH_NODES %}
 {%   set lsheap = LOGSTASH_MERGED.settings.lsheap %}
 
 include:
@@ -33,18 +28,23 @@ so-logstash:
       - sobridge:
         - ipv4_address: {{ DOCKER.containers['so-logstash'].ip }}
     - user: logstash
-    - extra_hosts: {{ REDIS_NODES }}
+    - extra_hosts:
+    {% for node in LOGSTASH_NODES %}
+    {%   for hostname, ip in node.items() %}
+      - {{hostname}}:{{ip}}
+    {%   endfor %}
+    {% endfor %}
     {% if DOCKER.containers['so-logstash'].extra_hosts %}
-      {% for XTRAHOST in DOCKER.containers['so-logstash'].extra_hosts %}
+    {%   for XTRAHOST in DOCKER.containers['so-logstash'].extra_hosts %}
       - {{ XTRAHOST }}
-      {% endfor %}
+    {%   endfor %}
     {% endif %}
     - environment:
       - LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }}
     {% if DOCKER.containers['so-logstash'].extra_env %}
-      {% for XTRAENV in DOCKER.containers['so-logstash'].extra_env %}
+    {%   for XTRAENV in DOCKER.containers['so-logstash'].extra_env %}
       - {{ XTRAENV }}
-      {% endfor %}
+    {%   endfor %}
     {% endif %}
     - port_bindings:
       {% for BINDING in DOCKER.containers['so-logstash'].port_bindings %}

salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja

@@ -3,7 +3,7 @@ output {
     if [metadata][pipeline] {
       if [metadata][_id] {
         elasticsearch {
-          hosts => "{{ GLOBALS.manager }}"
+          hosts => "{{ GLOBALS.hostname }}"
           ecs_compatibility => v8
           data_stream => true
           user => "{{ ES_USER }}"
@@ -17,7 +17,7 @@ output {
       }
       else {
         elasticsearch {
-          hosts => "{{ GLOBALS.manager }}"
+          hosts => "{{ GLOBALS.hostname }}"
           ecs_compatibility => v8
           data_stream => true
           user => "{{ ES_USER }}"
@@ -30,7 +30,7 @@ output {
     }
     else {
       elasticsearch {
-        hosts => "{{ GLOBALS.manager }}"
+        hosts => "{{ GLOBALS.hostname }}"
         ecs_compatibility => v8
         data_stream => true
         user => "{{ ES_USER }}"

salt/soc/defaults.yaml

@@ -470,6 +470,18 @@ soc:
         - rule.action
         - rule.reason
         - network.community_id
+      ':pfsense:':
+        - soc_timestamp
+        - source.ip
+        - source.port
+        - destination.ip
+        - destination.port
+        - network.transport
+        - network.direction
+        - observer.ingress.interface.name
+        - event.action
+        - event.reason
+        - network.community_id
       ':osquery:':
         - soc_timestamp
         - source.ip
@@ -1348,7 +1360,7 @@ soc:
               showSubtitle: true
             - name: Firewall
               description: Firewall events grouped by action
-              query: 'tags:firewall | groupby rule.action'
+              query: 'observer.type:firewall | groupby event.action'
               showSubtitle: true
         dashboards:
           advanced: true
@@ -1551,7 +1563,7 @@ soc:
               query: 'tags:s7* | groupby -sankey event.dataset source.ip destination.ip | groupby event.dataset | groupby source.ip | groupby destination.ip | groupby destination.port'
             - name: Firewall
               description: Firewall logs
-              query: 'tags:firewall | groupby -sankey rule.action interface.name | groupby rule.action | groupby interface.name | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port'
+              query: 'observer.type:firewall | groupby -sankey event.action observer.ingress.interface.name | groupby event.action | groupby observer.ingress.interface.name | groupby network.type | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port'
             - name: VLAN
               description: VLAN (Virtual Local Area Network) tagged logs
               query: '* AND _exists_:network.vlan.id | groupby network.vlan.id | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby event.dataset | groupby event.module | groupby observer.name | groupby source.geo.country_name | groupby destination.geo.country_name'

salt/soc/enabled.sls

@@ -33,21 +33,26 @@ so-soc:
       - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw
       - /opt/so/conf/soc/queue:/opt/sensoroni/queue:rw
       - /opt/so/saltstack:/opt/so/saltstack:rw
-    - extra_hosts: {{ DOCKER_EXTRA_HOSTS }}
-      {% if DOCKER.containers['so-soc'].extra_hosts %}
-        {% for XTRAHOST in DOCKER.containers['so-soc'].extra_hosts %}
+    - extra_hosts:
+    {% for node in DOCKER_EXTRA_HOSTS %}
+    {%   for hostname, ip in node.items() %}
+      - {{hostname}}:{{ip}}
+    {%   endfor %}
+    {% endfor %}
+    {% if DOCKER.containers['so-soc'].extra_hosts %}
+    {%   for XTRAHOST in DOCKER.containers['so-soc'].extra_hosts %}
       - {{ XTRAHOST }}
-        {% endfor %}
-      {% endif %}
+    {%   endfor %}
+    {% endif %}
     - port_bindings:
-      {% for BINDING in DOCKER.containers['so-soc'].port_bindings %}
+    {% for BINDING in DOCKER.containers['so-soc'].port_bindings %}
       - {{ BINDING }}
-      {% endfor %}
+    {% endfor %}
     {% if DOCKER.containers['so-soc'].extra_env %}
     - environment:
-      {% for XTRAENV in DOCKER.containers['so-soc'].extra_env %}
+    {%   for XTRAENV in DOCKER.containers['so-soc'].extra_env %}
       - {{ XTRAENV }}
-      {% endfor %}
+    {%   endfor %}
     {% endif %}
     - watch:
       - file: /opt/so/conf/soc/*

setup/so-verify

@@ -33,7 +33,7 @@ log_has_errors() {
     # Ignore Failed: 0 since that is the salt state output, and we detect state failures
     # via Result: False already.
 
-    # This is ignored for Ubuntu
+    # This is ignored for Ubuntu:
     # Failed to restart snapd.mounts-pre.target: Operation refused, unit snapd.mounts-pre.target
     # may be requested by dependency only (it is configured to refuse manual start/stop).
 
@@ -41,6 +41,10 @@ log_has_errors() {
 
     # "remove failed" is caused by a warning generated by upgrade of libwbclient
 
+    # Exit code 100 failure is likely apt-get running in the background, we wait for it to unlock.
+
+    # Failed to deduce dest mapping appears to occur when a shard isn't yet ready. Temporary.
+    
     grep -E "FAILED|Failed|failed|ERROR|Result: False|Error is not recoverable" "$setup_log" | \
         grep -vE "The Salt Master has cached the public key for this node" | \
         grep -vE "Minion failed to authenticate with the master" | \
@@ -58,6 +62,7 @@ log_has_errors() {
         grep -vE "remove failed" | \
         grep -vE "Failed to restart snapd" | \
         grep -vE "Login Failed Details" | \
+        grep -vE "Failed to deduce dest mappings" | \
         grep -vE "response from daemon: unauthorized" | \
         grep -vE "Reading first line of patchfile" | \
         grep -vE "Command failed with exit code" | \