diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json index 0979f98b6..492db03dc 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json @@ -27,6 +27,7 @@ } } } - } + }, + "force": true } {%- endraw -%} diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json index 32bff857b..5462dc861 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json @@ -28,6 +28,7 @@ } } } - } + }, + "force": true } {%- endraw -%} diff --git a/salt/elasticfleet/files/integrations-optional/sublime_platform.json b/salt/elasticfleet/files/integrations-optional/sublime_platform.json index 8feedc879..0116ed936 100644 --- a/salt/elasticfleet/files/integrations-optional/sublime_platform.json +++ b/salt/elasticfleet/files/integrations-optional/sublime_platform.json @@ -39,6 +39,7 @@ } } } - } + }, + "force": true } {%- endraw -%} diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json b/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json index f1774b905..b3d6c4609 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json @@ -16,5 +16,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json b/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json index b6a21249a..699c07ec3 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json @@ -73,5 +73,6 @@ "system-system/metrics": { "enabled": false } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json b/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json index b17986a53..8e9813741 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json @@ -63,5 +63,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json index 4c22f92ee..a2aaf5e0a 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json @@ -102,5 +102,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json index 29cc1a879..db4b1a8f6 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index baa8683ae..3b1a50560 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -28,5 +28,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json index 3b8cffcc1..c9b036e36 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json index b1fb71077..6a67c9c1c 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json index 0349c9fc3..5527607bd 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json @@ -16,5 +16,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json index 6b9cbffaf..b822421eb 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json @@ -72,5 +72,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json index 3aa740881..aa39c177b 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json index 840f36f6b..cc92092e9 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json index 60ee95f45..61ad057f4 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json index b789adc1d..a875e4bfc 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json index 089b5d4f8..89e9bbe8e 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json index a9d857b24..c3b04fd86 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json index 80baa45ca..4088f5a87 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json @@ -28,5 +28,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json index 653c788b5..ad32a6964 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json @@ -29,5 +29,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json index a5c4c3e81..8e6bf7958 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json @@ -36,5 +36,6 @@ "system-system/metrics": { "enabled": false } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json index b1454d4bd..b1d69a44f 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json @@ -16,5 +16,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json index 3df514f0b..6c42086bc 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json @@ -36,5 +36,6 @@ "system-system/metrics": { "enabled": false } - } + }, + "force": true } diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 6fe4635a3..818b0666c 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -26,7 +26,12 @@ so-elasticsearch: - networks: - sobridge: - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }} - - extra_hosts: {{ LOGSTASH_NODES }} + - extra_hosts: + {% for node in LOGSTASH_NODES %} + {% for hostname, ip in node.items() %} + - {{hostname}}:{{ip}} + {% endfor %} + {% endfor %} {% if DOCKER.containers['so-elasticsearch'].extra_hosts %} {% for XTRAHOST in DOCKER.containers['so-elasticsearch'].extra_hosts %} - {{ XTRAHOST }} diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index c76f81d21..dbb774d7f 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -8,12 +8,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} {% from 'logstash/map.jinja' import LOGSTASH_MERGED %} -{% from 'logstash/map.jinja' import REDIS_NODES %} -{# we append the manager here so that it is added to extra_hosts so the heavynode can resolve it #} -{# we cannont append in the logstash/map.jinja because then it would be added to the 0900_input_redis.conf #} -{% if GLOBALS.role == 'so-heavynode' %} -{% do REDIS_NODES.append({GLOBALS.manager:GLOBALS.manager_ip}) %} -{% endif %} +{% from 'logstash/map.jinja' import LOGSTASH_NODES %} {% set lsheap = LOGSTASH_MERGED.settings.lsheap %} include: @@ -33,18 +28,23 @@ so-logstash: - sobridge: - ipv4_address: {{ DOCKER.containers['so-logstash'].ip }} - user: logstash - - extra_hosts: {{ REDIS_NODES }} + - extra_hosts: + {% for node in LOGSTASH_NODES %} + {% for hostname, ip in node.items() %} + - {{hostname}}:{{ip}} + {% endfor %} + {% endfor %} {% if DOCKER.containers['so-logstash'].extra_hosts %} - {% for XTRAHOST in DOCKER.containers['so-logstash'].extra_hosts %} + {% for XTRAHOST in DOCKER.containers['so-logstash'].extra_hosts %} - {{ XTRAHOST }} - {% endfor %} + {% endfor %} {% endif %} - environment: - LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }} {% if DOCKER.containers['so-logstash'].extra_env %} - {% for XTRAENV in DOCKER.containers['so-logstash'].extra_env %} + {% for XTRAENV in DOCKER.containers['so-logstash'].extra_env %} - {{ XTRAENV }} - {% endfor %} + {% endfor %} {% endif %} - port_bindings: {% for BINDING in DOCKER.containers['so-logstash'].port_bindings %} diff --git a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja index f7671e2b7..3a86cd8be 100644 --- a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja +++ b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja @@ -3,7 +3,7 @@ output { if [metadata][pipeline] { if [metadata][_id] { elasticsearch { - hosts => "{{ GLOBALS.manager }}" + hosts => "{{ GLOBALS.hostname }}" ecs_compatibility => v8 data_stream => true user => "{{ ES_USER }}" @@ -17,7 +17,7 @@ output { } else { elasticsearch { - hosts => "{{ GLOBALS.manager }}" + hosts => "{{ GLOBALS.hostname }}" ecs_compatibility => v8 data_stream => true user => "{{ ES_USER }}" @@ -30,7 +30,7 @@ output { } else { elasticsearch { - hosts => "{{ GLOBALS.manager }}" + hosts => "{{ GLOBALS.hostname }}" ecs_compatibility => v8 data_stream => true user => "{{ ES_USER }}" diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 3655019e4..f440bd57b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -470,6 +470,18 @@ soc: - rule.action - rule.reason - network.community_id + ':pfsense:': + - soc_timestamp + - source.ip + - source.port + - destination.ip + - destination.port + - network.transport + - network.direction + - observer.ingress.interface.name + - event.action + - event.reason + - network.community_id ':osquery:': - soc_timestamp - source.ip @@ -1348,7 +1360,7 @@ soc: showSubtitle: true - name: Firewall description: Firewall events grouped by action - query: 'tags:firewall | groupby rule.action' + query: 'observer.type:firewall | groupby event.action' showSubtitle: true dashboards: advanced: true @@ -1551,7 +1563,7 @@ soc: query: 'tags:s7* | groupby -sankey event.dataset source.ip destination.ip | groupby event.dataset | groupby source.ip | groupby destination.ip | groupby destination.port' - name: Firewall description: Firewall logs - query: 'tags:firewall | groupby -sankey rule.action interface.name | groupby rule.action | groupby interface.name | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'observer.type:firewall | groupby -sankey event.action observer.ingress.interface.name | groupby event.action | groupby observer.ingress.interface.name | groupby network.type | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port' - name: VLAN description: VLAN (Virtual Local Area Network) tagged logs query: '* AND _exists_:network.vlan.id | groupby network.vlan.id | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby event.dataset | groupby event.module | groupby observer.name | groupby source.geo.country_name | groupby destination.geo.country_name' diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 9e588e5b1..2661587f4 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -33,21 +33,26 @@ so-soc: - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw - /opt/so/conf/soc/queue:/opt/sensoroni/queue:rw - /opt/so/saltstack:/opt/so/saltstack:rw - - extra_hosts: {{ DOCKER_EXTRA_HOSTS }} - {% if DOCKER.containers['so-soc'].extra_hosts %} - {% for XTRAHOST in DOCKER.containers['so-soc'].extra_hosts %} + - extra_hosts: + {% for node in DOCKER_EXTRA_HOSTS %} + {% for hostname, ip in node.items() %} + - {{hostname}}:{{ip}} + {% endfor %} + {% endfor %} + {% if DOCKER.containers['so-soc'].extra_hosts %} + {% for XTRAHOST in DOCKER.containers['so-soc'].extra_hosts %} - {{ XTRAHOST }} - {% endfor %} - {% endif %} + {% endfor %} + {% endif %} - port_bindings: - {% for BINDING in DOCKER.containers['so-soc'].port_bindings %} + {% for BINDING in DOCKER.containers['so-soc'].port_bindings %} - {{ BINDING }} - {% endfor %} + {% endfor %} {% if DOCKER.containers['so-soc'].extra_env %} - environment: - {% for XTRAENV in DOCKER.containers['so-soc'].extra_env %} + {% for XTRAENV in DOCKER.containers['so-soc'].extra_env %} - {{ XTRAENV }} - {% endfor %} + {% endfor %} {% endif %} - watch: - file: /opt/so/conf/soc/* diff --git a/setup/so-verify b/setup/so-verify index 3c20d22b4..6f47940ac 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -33,13 +33,17 @@ log_has_errors() { # Ignore Failed: 0 since that is the salt state output, and we detect state failures # via Result: False already. - # This is ignored for Ubuntu + # This is ignored for Ubuntu: # Failed to restart snapd.mounts-pre.target: Operation refused, unit snapd.mounts-pre.target # may be requested by dependency only (it is configured to refuse manual start/stop). # Command failed with exit code is output during retry loops. # "remove failed" is caused by a warning generated by upgrade of libwbclient + + # Exit code 100 failure is likely apt-get running in the background, we wait for it to unlock. + + # Failed to deduce dest mapping appears to occur when a shard isn't yet ready. Temporary. grep -E "FAILED|Failed|failed|ERROR|Result: False|Error is not recoverable" "$setup_log" | \ grep -vE "The Salt Master has cached the public key for this node" | \ @@ -58,6 +62,7 @@ log_has_errors() { grep -vE "remove failed" | \ grep -vE "Failed to restart snapd" | \ grep -vE "Login Failed Details" | \ + grep -vE "Failed to deduce dest mappings" | \ grep -vE "response from daemon: unauthorized" | \ grep -vE "Reading first line of patchfile" | \ grep -vE "Command failed with exit code" | \