CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-06-12 13:19:22 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15781 from Security-Onion-Solutions/feature/postgres

Browse Source 
Telegraf Postgres: shared schema + JSONB storage
This commit is contained in:
Mike Reeves
2026-04-16 17:29:29 -04:00
committed by GitHub
parent a5e5f12889 d9a9029ce5
commit 9cce920d78
57 changed files with 1220 additions and 521 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pillar/elasticsearch/index_templates.sls
-2
View File
@@ -1,2 +0,0 @@
elasticsearch:
index_settings:
pillar/top.sls
-3
View File
@@ -102,7 +102,6 @@ base:
- node_data.ips
- secrets
- healthcheck.eval
- elasticsearch.index_templates
{% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
- elasticsearch.auth
{% endif %}
@@ -152,7 +151,6 @@ base:
- logstash.nodes
- logstash.soc_logstash
- logstash.adv_logstash
- elasticsearch.index_templates
{% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
- elasticsearch.auth
{% endif %}
@@ -271,7 +269,6 @@ base:
'*_import':
- node_data.ips
- secrets
- elasticsearch.index_templates
{% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
- elasticsearch.auth
{% endif %}
salt/elasticfleet/content-defaults.map.jinja
+123
View File
@@ -0,0 +1,123 @@
{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
this file except in compliance with the Elastic License 2.0. #}
{% import_json '/opt/so/state/esfleet_content_package_components.json' as ADDON_CONTENT_PACKAGE_COMPONENTS %}
{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
{% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %}
{% set DEBUG_STUFF = {} %}
{% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %}
{% if pkg.name in CORE_ESFLEET_PACKAGES %}
{# skip core content packages #}
{% elif pkg.name not in CORE_ESFLEET_PACKAGES %}
{# generate defaults for each content package #}
{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0%}
{% for pattern in pkg.dataStreams %}
{# in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
title: generic is an artifact of that and is not in use #}
{% if pattern.title == "generic" %}
{% continue %}
{% endif %}
{% if "metrics-" in pattern.name %}
{% set integration_type = "metrics-" %}
{% elif "logs-" in pattern.name %}
{% set integration_type = "logs-" %}
{% else %}
{% set integration_type = "" %}
{% endif %}
{# on content integrations the component name is user defined at the time it is added to an agent policy #}
{% set component_name = pattern.title %}
{% set index_pattern = pattern.name %}
{# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
{% set component_name_x = component_name.replace(".","_x_") %}
{# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
{% set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
{# Default integration settings #}
{% set integration_defaults = {
"index_sorting": false,
"index_template": {
"composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
"index_patterns": [index_pattern],
"priority": 501,
"template": {
"settings": {
"index": {
"lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
"number_of_replicas": 0
}
}
}
},
"policy": {
"phases": {
"cold": {
"actions": {
"allocate":{
"number_of_replicas": ""
},
"set_priority": {"priority": 0}
},
"min_age": "60d"
},
"delete": {
"actions": {
"delete": {}
},
"min_age": "365d"
},
"hot": {
"actions": {
"rollover": {
"max_age": "30d",
"max_primary_shard_size": "50gb"
},
"forcemerge":{
"max_num_segments": ""
},
"shrink":{
"max_primary_shard_size": "",
"method": "COUNT",
"number_of_shards": ""
},
"set_priority": {"priority": 100}
},
"min_age": "0ms"
},
"warm": {
"actions": {
"allocate": {
"number_of_replicas": ""
},
"forcemerge": {
"max_num_segments": ""
},
"shrink":{
"max_primary_shard_size": "",
"method": "COUNT",
"number_of_shards": ""
},
"set_priority": {"priority": 50}
},
"min_age": "30d"
}
}
}
} %}
{% do ADDON_CONTENT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
{% endfor %}
{% else %}
{% endif %}
{% endif %}
{% endfor %}
salt/elasticfleet/defaults.yaml
+1
View File
@@ -1,5 +1,6 @@
elasticfleet:
enabled: False
patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
enable_manager_output: True
config:
server:
salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json
+9 -2
View File
@@ -9,16 +9,22 @@
"namespace": "so",
"description": "Zeek Import logs",
"policy_id": "so-grid-nodes_general",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/nsm/import/*/zeek/logs/*.log"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "import",
"pipeline": "",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -34,7 +40,8 @@
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json
+10 -4
View File
@@ -15,19 +15,25 @@
"version": ""
},
"name": "kratos-logs",
"namespace": "so",
"description": "Kratos logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/kratos/kratos.log"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "kratos",
"pipeline": "kratos",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -48,10 +54,10 @@
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json
+9 -3
View File
@@ -9,16 +9,22 @@
"namespace": "so",
"description": "Zeek logs",
"policy_id": "so-grid-nodes_general",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/nsm/zeek/logs/current/*.log"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "zeek",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
@@ -30,10 +36,10 @@
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json
+1 -1
View File
@@ -5,7 +5,7 @@
"package": {
"name": "endpoint",
"title": "Elastic Defend",
"version": "9.0.2",
"version": "9.3.0",
"requires_root": true
},
"enabled": true,
salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json
+10 -7
View File
@@ -6,21 +6,23 @@
"name": "agent-monitor",
"namespace": "",
"description": "",
"policy_id": "so-grid-nodes_general",
"policy_ids": [
"so-grid-nodes_general"
],
"output_id": null,
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/agents/agent-monitor.log"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "agentmonitor",
"pipeline": "elasticagent.monitor",
"parsers": "",
@@ -34,15 +36,16 @@
"ignore_older": "72h",
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": true,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": 64,
"file_identity_native": false,
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
}
}
},
"force": true
}
salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json
+10 -4
View File
@@ -4,19 +4,25 @@
"version": ""
},
"name": "hydra-logs",
"namespace": "so",
"description": "Hydra logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/hydra/hydra.log"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "hydra",
"pipeline": "hydra",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -34,10 +40,10 @@
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json
+10 -4
View File
@@ -4,19 +4,25 @@
"version": ""
},
"name": "idh-logs",
"namespace": "so",
"description": "IDH integration",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/nsm/idh/opencanary.log"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "idh",
"pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -31,10 +37,10 @@
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json
+11 -5
View File
@@ -4,26 +4,32 @@
"version": ""
},
"name": "import-evtx-logs",
"namespace": "so",
"description": "Import Windows EVTX logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/nsm/import/*/evtx/*.json"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "import",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
"\\.gz$"
],
"include_files": [],
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.6.1\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.6.1\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.6.1\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n target: data_stream\n fields:\n dataset: import",
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.15.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.15.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.15.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import",
"tags": [
"import"
],
@@ -33,10 +39,10 @@
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json
+10 -4
View File
@@ -4,19 +4,25 @@
"version": ""
},
"name": "import-suricata-logs",
"namespace": "so",
"description": "Import Suricata logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/nsm/import/*/suricata/eve*.json"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "import",
"pipeline": "suricata.common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -32,10 +38,10 @@
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json
+10 -4
View File
@@ -4,14 +4,18 @@
"version": ""
},
"name": "rita-logs",
"namespace": "so",
"description": "RITA Logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
@@ -19,6 +23,8 @@
"/nsm/rita/exploded-dns.csv",
"/nsm/rita/long-connections.csv"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "rita",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
@@ -33,10 +39,10 @@
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json
+10 -4
View File
@@ -4,19 +4,25 @@
"version": ""
},
"name": "so-ip-mappings",
"namespace": "so",
"description": "IP Description mappings",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/nsm/custom-mappings/ip-descriptions.csv"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "hostnamemappings",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
@@ -32,10 +38,10 @@
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json
+10 -4
View File
@@ -4,19 +4,25 @@
"version": ""
},
"name": "soc-auth-sync-logs",
"namespace": "so",
"description": "Security Onion - Elastic Auth Sync - Logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/soc/sync.log"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "soc",
"pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -31,10 +37,10 @@
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json
+10 -4
View File
@@ -4,20 +4,26 @@
"version": ""
},
"name": "soc-detections-logs",
"namespace": "so",
"description": "Security Onion Console - Detections Logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/soc/detections_runtime-status_sigma.log",
"/opt/so/log/soc/detections_runtime-status_yara.log"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "soc",
"pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -35,10 +41,10 @@
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json
+10 -4
View File
@@ -4,19 +4,25 @@
"version": ""
},
"name": "soc-salt-relay-logs",
"namespace": "so",
"description": "Security Onion - Salt Relay - Logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/soc/salt-relay.log"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "soc",
"pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -33,10 +39,10 @@
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json
+10 -4
View File
@@ -4,19 +4,25 @@
"version": ""
},
"name": "soc-sensoroni-logs",
"namespace": "so",
"description": "Security Onion - Sensoroni - Logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/sensoroni/sensoroni.log"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "soc",
"pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -31,10 +37,10 @@
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json
+10 -4
View File
@@ -4,19 +4,25 @@
"version": ""
},
"name": "soc-server-logs",
"namespace": "so",
"description": "Security Onion Console Logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/soc/sensoroni-server.log"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "soc",
"pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -33,10 +39,10 @@
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json
+10 -4
View File
@@ -4,19 +4,25 @@
"version": ""
},
"name": "strelka-logs",
"namespace": "so",
"description": "Strelka Logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/nsm/strelka/log/strelka.log"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "strelka",
"pipeline": "strelka.file",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -31,10 +37,10 @@
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json
+10 -4
View File
@@ -4,19 +4,25 @@
"version": ""
},
"name": "suricata-logs",
"namespace": "so",
"description": "Suricata integration",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"policy_ids": [
"so-grid-nodes_general"
],
"vars": {},
"inputs": {
"filestream-filestream": {
"enabled": true,
"streams": {
"filestream.generic": {
"filestream.filestream": {
"enabled": true,
"vars": {
"paths": [
"/nsm/suricata/eve*.json"
],
"compression_gzip": false,
"use_logs_stream": false,
"data_stream.dataset": "suricata",
"pipeline": "suricata.common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
@@ -31,10 +37,10 @@
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
"include_lines": [],
"delete_enabled": false
}
}
}
salt/elasticfleet/input-defaults.map.jinja
+123
View File
@@ -0,0 +1,123 @@
{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
this file except in compliance with the Elastic License 2.0. #}
{% import_json '/opt/so/state/esfleet_input_package_components.json' as ADDON_INPUT_PACKAGE_COMPONENTS %}
{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
{% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %}
{% set DEBUG_STUFF = {} %}
{% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %}
{% if pkg.name in CORE_ESFLEET_PACKAGES %}
{# skip core input packages #}
{% elif pkg.name not in CORE_ESFLEET_PACKAGES %}
{# generate defaults for each input package #}
{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
{% for pattern in pkg.dataStreams %}
{# in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
title: generic is an artifact of that and is not in use #}
{% if pattern.title == "generic" %}
{% continue %}
{% endif %}
{% if "metrics-" in pattern.name %}
{% set integration_type = "metrics-" %}
{% elif "logs-" in pattern.name %}
{% set integration_type = "logs-" %}
{% else %}
{% set integration_type = "" %}
{% endif %}
{# on input integrations the component name is user defined at the time it is added to an agent policy #}
{% set component_name = pattern.title %}
{% set index_pattern = pattern.name %}
{# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
{% set component_name_x = component_name.replace(".","_x_") %}
{# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
{% set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
{# Default integration settings #}
{% set integration_defaults = {
"index_sorting": false,
"index_template": {
"composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
"index_patterns": [index_pattern],
"priority": 501,
"template": {
"settings": {
"index": {
"lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
"number_of_replicas": 0
}
}
}
},
"policy": {
"phases": {
"cold": {
"actions": {
"allocate":{
"number_of_replicas": ""
},
"set_priority": {"priority": 0}
},
"min_age": "60d"
},
"delete": {
"actions": {
"delete": {}
},
"min_age": "365d"
},
"hot": {
"actions": {
"rollover": {
"max_age": "30d",
"max_primary_shard_size": "50gb"
},
"forcemerge":{
"max_num_segments": ""
},
"shrink":{
"max_primary_shard_size": "",
"method": "COUNT",
"number_of_shards": ""
},
"set_priority": {"priority": 100}
},
"min_age": "0ms"
},
"warm": {
"actions": {
"allocate": {
"number_of_replicas": ""
},
"forcemerge": {
"max_num_segments": ""
},
"shrink":{
"max_primary_shard_size": "",
"method": "COUNT",
"number_of_shards": ""
},
"set_priority": {"priority": 50}
},
"min_age": "30d"
}
}
}
} %}
{% do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
{% do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%}
{% endfor %}
{% endif %}
{% endif %}
{% endfor %}
salt/elasticfleet/integration-defaults.map.jinja
+5 -22
View File
@@ -59,8 +59,8 @@
{# skip core integrations #}
{% elif pkg.name not in CORE_ESFLEET_PACKAGES %}
{# generate defaults for each integration #}
{% if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %}
{% for pattern in pkg.es_index_patterns %}
{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
{% for pattern in pkg.dataStreams %}
{% if "metrics-" in pattern.name %}
{% set integration_type = "metrics-" %}
{% elif "logs-" in pattern.name %}
@@ -75,44 +75,27 @@
{% if component_name in WEIRD_INTEGRATIONS %}
{% set component_name = WEIRD_INTEGRATIONS[component_name] %}
{% endif %}
{# create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed
eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #}
{% set custom_component_name = component_name %}
{# duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #}
{% set generic_integration_type = integration_type %}
{# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
{% set component_name_x = component_name.replace(".","_x_") %}
{# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
{% set integration_key = "so-" ~ integration_type ~ component_name_x %}
{# if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #}
{% if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %}
{# these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
{% set index_pattern = integration_type ~ component_name ~ "-*" %}
{# includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
{% set component_name = "filestream.generic" %}
{% set generic_integration_type = "logs-" %}
{% endif %}
{# Default integration settings #}
{% set integration_defaults = {
"index_sorting": false,
"index_template": {
"composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
"composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
"data_stream": {
"allow_custom_routing": false,
"hidden": false
},
"ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"],
"ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
"index_patterns": [index_pattern],
"priority": 501,
"template": {
"settings": {
"index": {
"lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"},
"lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
"number_of_replicas": 0
}
}
salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+26 -2
View File
@@ -135,9 +135,33 @@ elastic_fleet_bulk_package_install() {
fi
}
elastic_fleet_installed_packages() {
if ! fleet_api "epm/packages/installed?perPage=500"; then
elastic_fleet_get_package_list_by_type() {
if ! output=$(fleet_api "epm/packages"); then
return 1
else
is_integration=$(jq '[.items[] | select(.type=="integration") | .name ]' <<< "$output")
is_input=$(jq '[.items[] | select(.type=="input") | .name ]' <<< "$output")
is_content=$(jq '[.items[] | select(.type=="content") | .name ]' <<< "$output")
jq -n --argjson is_integration "${is_integration:-[]}" \
--argjson is_input "${is_input:-[]}" \
--argjson is_content "${is_content:-[]}" \
'{"integration": $is_integration,"input": $is_input, "content": $is_content}'
fi
}
elastic_fleet_installed_packages_components() {
package_type=${1,,}
if [[ "$package_type" != "integration" && "$package_type" != "input" && "$package_type" != "content" ]]; then
echo "Error: Invalid package type ${package_type}. Valid types are 'integration', 'input', or 'content'."
return 1
fi
packages_by_type=$(elastic_fleet_get_package_list_by_type)
packages=$(jq --arg package_type "$package_type" '.[$package_type]' <<< "$packages_by_type")
if ! output=$(fleet_api "epm/packages/installed?perPage=500"); then
return 1
else
jq -c --argjson packages "$packages" '[.items[] | select(.name | IN($packages[])) | {name: .name, dataStreams: .dataStreams}]' <<< "$output"
fi
}
salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade
+5
View File
@@ -6,6 +6,11 @@
. /usr/sbin/so-common
{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
{# Optionally override Elasticsearch version for Elastic Agent patch releases #}
{%- if ELASTICFLEETDEFAULTS.elasticfleet.patch_version is defined %}
{%- do ELASTICSEARCHDEFAULTS.update({'elasticsearch': {'version': ELASTICFLEETDEFAULTS.elasticfleet.patch_version}}) %}
{%- endif %}
# Only run on Managers
if ! is_manager_node; then
salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
+10 -5
View File
@@ -18,7 +18,9 @@ INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json
CONTENT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_content_package_components.json
COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json
PENDING_UPDATE=false
@@ -179,10 +181,13 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then
else
echo "Elastic integrations don't appear to need installation/updating..."
fi
# Write out file for generating index/component/ilm templates
if latest_installed_package_list=$(elastic_fleet_installed_packages); then
echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
fi
# Write out file for generating index/component/ilm templates, keeping each package type separate
for package_type in "INTEGRATION" "INPUT" "CONTENT"; do
if latest_installed_package_list=$(elastic_fleet_installed_packages_components "$package_type"); then
outfile="${package_type}_PACKAGE_COMPONENTS"
echo $latest_installed_package_list > "${!outfile}"
fi
done
if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then
# Refresh installed component template list
latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')
salt/elasticsearch/config.sls
+9
View File
@@ -66,6 +66,8 @@ so-elasticsearch-ilm-policy-load-script:
- group: 939
- mode: 754
- template: jinja
- defaults:
GLOBALS: {{ GLOBALS }}
- show_changes: False
so-elasticsearch-pipelines-script:
@@ -91,6 +93,13 @@ estemplatedir:
- group: 939
- makedirs: True
esaddontemplatedir:
file.directory:
- name: /opt/so/conf/elasticsearch/templates/addon-index
- user: 930
- group: 939
- makedirs: True
esrolesdir:
file.directory:
- name: /opt/so/conf/elasticsearch/roles
salt/elasticsearch/defaults.yaml
+1 -1
View File
@@ -1,6 +1,6 @@
elasticsearch:
enabled: false
version: 9.0.8
version: 9.3.3
index_clean: true
vm:
max_map_count: 1048576
salt/elasticsearch/enabled.sls
+45 -21
View File
@@ -10,8 +10,10 @@
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
{% if GLOBALS.role != 'so-heavynode' %}
{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
{% endif %}
include:
- ca
@@ -117,40 +119,52 @@ escomponenttemplates:
- onchanges_in:
- file: so-elasticsearch-templates-reload
- show_changes: False
# Auto-generate templates from defaults file
# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory
so_index_template_dir:
file.directory:
- name: /opt/so/conf/elasticsearch/templates/index
- clean: True
{%- if SO_MANAGED_INDICES %}
- require:
{%- for index in SO_MANAGED_INDICES %}
- file: so_index_template_{{index}}
{%- endfor %}
{%- endif %}
# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml)
# These index templates are for the core SO datasets and are always required
{% for index, settings in ES_INDEX_SETTINGS.items() %}
{% if settings.index_template is defined %}
es_index_template_{{index}}:
{% if settings.index_template is defined %}
so_index_template_{{index}}:
file.managed:
- name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
- source: salt://elasticsearch/base-template.json.jinja
- defaults:
TEMPLATE_CONFIG: {{ settings.index_template }}
TEMPLATE_CONFIG: {{ settings.index_template }}
- template: jinja
- show_changes: False
- onchanges_in:
- file: so-elasticsearch-templates-reload
{% endif %}
{% endfor %}
{% if TEMPLATES %}
# Sync custom templates to /opt/so/conf/elasticsearch/templates
{% for TEMPLATE in TEMPLATES %}
es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
{% if GLOBALS.role != "so-heavynode" %}
# Auto-generate optional index templates for integration | input | content packages
# These index templates are not used by default (until user adds package to an agent policy).
# Pre-configured with standard defaults, and incorporated into SOC configuration for user customization.
{% for index,settings in ALL_ADDON_SETTINGS.items() %}
{% if settings.index_template is defined %}
addon_index_template_{{index}}:
file.managed:
- source: salt://elasticsearch/templates/index/{{TEMPLATE}}
{% if 'jinja' in TEMPLATE.split('.')[-1] %}
- name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}
- name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json
- source: salt://elasticsearch/base-template.json.jinja
- defaults:
TEMPLATE_CONFIG: {{ settings.index_template }}
- template: jinja
{% else %}
- name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}}
{% endif %}
- user: 930
- group: 939
- show_changes: False
- onchanges_in:
- file: so-elasticsearch-templates-reload
- file: addon-elasticsearch-templates-reload
{% endif %}
{% endfor %}
{% endif %}
@@ -165,6 +179,7 @@ so-es-cluster-settings:
- file: elasticsearch_sbin_jinja
{% endif %}
# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
so-elasticsearch-ilm-policy-load:
cmd.run:
- name: /usr/sbin/so-elasticsearch-ilm-policy-load
@@ -179,9 +194,18 @@ so-elasticsearch-templates-reload:
file.absent:
- name: /opt/so/state/estemplates.txt
addon-elasticsearch-templates-reload:
file.absent:
- name: /opt/so/state/addon_estemplates.txt
# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script
so-elasticsearch-templates:
cmd.run:
{%- if GLOBALS.role == "so-heavynode" %}
- name: /usr/sbin/so-elasticsearch-templates-load --heavynode
{%- else %}
- name: /usr/sbin/so-elasticsearch-templates-load
{%- endif %}
- cwd: /opt/so
- template: jinja
- require:
salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1 → salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2
+74 -13
View File
@@ -10,24 +10,28 @@
"processors": [
{
"set": {
"tag": "set_ecs_version_f5923549",
"field": "ecs.version",
"value": "8.17.0"
}
},
{
"set": {
"tag": "set_observer_vendor_ad9d35cc",
"field": "observer.vendor",
"value": "netgate"
}
},
{
"set": {
"tag": "set_observer_type_5dddf3ba",
"field": "observer.type",
"value": "firewall"
}
},
{
"rename": {
"tag": "rename_message_to_event_original_56a77271",
"field": "message",
"target_field": "event.original",
"ignore_missing": true,
@@ -36,12 +40,14 @@
},
{
"set": {
"tag": "set_event_kind_de80643c",
"field": "event.kind",
"value": "event"
}
},
{
"set": {
"tag": "set_event_timezone_4ca44cac",
"field": "event.timezone",
"value": "{{{_tmp.tz_offset}}}",
"if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'"
@@ -49,6 +55,7 @@
},
{
"grok": {
"tag": "grok_event_original_27d9c8c7",
"description": "Parse syslog header",
"field": "event.original",
"patterns": [
@@ -72,6 +79,7 @@
},
{
"date": {
"tag": "date__tmp_timestamp8601_to_timestamp_6ac9d3ce",
"if": "ctx._tmp.timestamp8601 != null",
"field": "_tmp.timestamp8601",
"target_field": "@timestamp",
@@ -82,6 +90,7 @@
},
{
"date": {
"tag": "date__tmp_timestamp_to_timestamp_f21e536e",
"if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null",
"field": "_tmp.timestamp",
"target_field": "@timestamp",
@@ -95,6 +104,7 @@
},
{
"grok": {
"tag": "grok_process_name_cef3d489",
"description": "Set Event Provider",
"field": "process.name",
"patterns": [
@@ -107,71 +117,83 @@
},
{
"pipeline": {
"name": "logs-pfsense.log-1.23.1-firewall",
"tag": "pipeline_e16851a7",
"name": "logs-pfsense.log-1.25.2-firewall",
"if": "ctx.event.provider == 'filterlog'"
}
},
{
"pipeline": {
"name": "logs-pfsense.log-1.23.1-openvpn",
"tag": "pipeline_828590b5",
"name": "logs-pfsense.log-1.25.2-openvpn",
"if": "ctx.event.provider == 'openvpn'"
}
},
{
"pipeline": {
"name": "logs-pfsense.log-1.23.1-ipsec",
"tag": "pipeline_9d37039c",
"name": "logs-pfsense.log-1.25.2-ipsec",
"if": "ctx.event.provider == 'charon'"
}
},
{
"pipeline": {
"name": "logs-pfsense.log-1.23.1-dhcp",
"if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
"tag": "pipeline_ad56bbca",
"name": "logs-pfsense.log-1.25.2-dhcp",
"if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)"
}
},
{
"pipeline": {
"name": "logs-pfsense.log-1.23.1-unbound",
"tag": "pipeline_dd85553d",
"name": "logs-pfsense.log-1.25.2-unbound",
"if": "ctx.event.provider == 'unbound'"
}
},
{
"pipeline": {
"name": "logs-pfsense.log-1.23.1-haproxy",
"tag": "pipeline_720ed255",
"name": "logs-pfsense.log-1.25.2-haproxy",
"if": "ctx.event.provider == 'haproxy'"
}
},
{
"pipeline": {
"name": "logs-pfsense.log-1.23.1-php-fpm",
"tag": "pipeline_456beba5",
"name": "logs-pfsense.log-1.25.2-php-fpm",
"if": "ctx.event.provider == 'php-fpm'"
}
},
{
"pipeline": {
"name": "logs-pfsense.log-1.23.1-squid",
"tag": "pipeline_a0d89375",
"name": "logs-pfsense.log-1.25.2-squid",
"if": "ctx.event.provider == 'squid'"
}
},
{
"pipeline": {
"name": "logs-pfsense.log-1.23.1-snort",
"tag": "pipeline_c2f1ed55",
"name": "logs-pfsense.log-1.25.2-snort",
"if": "ctx.event.provider == 'snort'"
}
},
{
"pipeline": {
"name": "logs-pfsense.log-1.23.1-suricata",
"tag":"pipeline_33db1c9e",
"name": "logs-pfsense.log-1.25.2-suricata",
"if": "ctx.event.provider == 'suricata'"
}
},
{
"drop": {
"if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
"tag": "drop_9d7c46f8",
"if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dnsmasq-dhcp\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
}
},
{
"append": {
"tag": "append_event_category_4780a983",
"field": "event.category",
"value": "network",
"if": "ctx.network != null"
@@ -179,6 +201,7 @@
},
{
"convert": {
"tag": "convert_source_address_to_source_ip_f5632a20",
"field": "source.address",
"target_field": "source.ip",
"type": "ip",
@@ -188,6 +211,7 @@
},
{
"convert": {
"tag": "convert_destination_address_to_destination_ip_f1388f0c",
"field": "destination.address",
"target_field": "destination.ip",
"type": "ip",
@@ -197,6 +221,7 @@
},
{
"set": {
"tag": "set_network_type_1f1d940a",
"field": "network.type",
"value": "ipv6",
"if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")"
@@ -204,6 +229,7 @@
},
{
"set": {
"tag": "set_network_type_69deca38",
"field": "network.type",
"value": "ipv4",
"if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")"
@@ -211,6 +237,7 @@
},
{
"geoip": {
"tag": "geoip_source_ip_to_source_geo_da2e41b2",
"field": "source.ip",
"target_field": "source.geo",
"ignore_missing": true
@@ -218,6 +245,7 @@
},
{
"geoip": {
"tag": "geoip_destination_ip_to_destination_geo_ab5e2968",
"field": "destination.ip",
"target_field": "destination.geo",
"ignore_missing": true
@@ -225,6 +253,7 @@
},
{
"geoip": {
"tag": "geoip_source_ip_to_source_as_28d69883",
"ignore_missing": true,
"database_file": "GeoLite2-ASN.mmdb",
"field": "source.ip",
@@ -237,6 +266,7 @@
},
{
"geoip": {
"tag": "geoip_destination_ip_to_destination_as_8a007787",
"database_file": "GeoLite2-ASN.mmdb",
"field": "destination.ip",
"target_field": "destination.as",
@@ -249,6 +279,7 @@
},
{
"rename": {
"tag": "rename_source_as_asn_to_source_as_number_a917047d",
"field": "source.as.asn",
"target_field": "source.as.number",
"ignore_missing": true
@@ -256,6 +287,7 @@
},
{
"rename": {
"tag": "rename_source_as_organization_name_to_source_as_organization_name_f1362d0b",
"field": "source.as.organization_name",
"target_field": "source.as.organization.name",
"ignore_missing": true
@@ -263,6 +295,7 @@
},
{
"rename": {
"tag": "rename_destination_as_asn_to_destination_as_number_3b459fcd",
"field": "destination.as.asn",
"target_field": "destination.as.number",
"ignore_missing": true
@@ -270,6 +303,7 @@
},
{
"rename": {
"tag": "rename_destination_as_organization_name_to_destination_as_organization_name_814bd459",
"field": "destination.as.organization_name",
"target_field": "destination.as.organization.name",
"ignore_missing": true
@@ -277,12 +311,14 @@
},
{
"community_id": {
"tag": "community_id_d2308e7a",
"target_field": "network.community_id",
"ignore_failure": true
}
},
{
"grok": {
"tag": "grok_observer_ingress_interface_name_968018d3",
"field": "observer.ingress.interface.name",
"patterns": [
"%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}"
@@ -293,6 +329,7 @@
},
{
"set": {
"tag": "set_network_vlan_id_efd4d96a",
"field": "network.vlan.id",
"copy_from": "observer.ingress.vlan.id",
"ignore_empty_value": true
@@ -300,6 +337,7 @@
},
{
"append": {
"tag": "append_related_ip_c1a6356b",
"field": "related.ip",
"value": "{{{destination.ip}}}",
"allow_duplicates": false,
@@ -308,6 +346,7 @@
},
{
"append": {
"tag": "append_related_ip_8121c591",
"field": "related.ip",
"value": "{{{source.ip}}}",
"allow_duplicates": false,
@@ -316,6 +355,7 @@
},
{
"append": {
"tag": "append_related_ip_53b62ed8",
"field": "related.ip",
"value": "{{{source.nat.ip}}}",
"allow_duplicates": false,
@@ -324,6 +364,7 @@
},
{
"append": {
"tag": "append_related_hosts_6f162628",
"field": "related.hosts",
"value": "{{{destination.domain}}}",
"if": "ctx.destination?.domain != null"
@@ -331,6 +372,7 @@
},
{
"append": {
"tag": "append_related_user_c036eec2",
"field": "related.user",
"value": "{{{user.name}}}",
"if": "ctx.user?.name != null"
@@ -338,6 +380,7 @@
},
{
"set": {
"tag": "set_network_direction_cb1e3125",
"field": "network.direction",
"value": "{{{network.direction}}}bound",
"if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/"
@@ -345,6 +388,7 @@
},
{
"remove": {
"tag": "remove_a82e20f2",
"field": [
"_tmp"
],
@@ -353,11 +397,21 @@
},
{
"script": {
"tag": "script_a7f2c062",
"lang": "painless",
"description": "This script processor iterates over the whole document to remove fields with null values.",
"source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n"
}
},
{
"append": {
"tag": "append_preserve_original_event_on_error",
"field": "tags",
"value": "preserve_original_event",
"allow_duplicates": false,
"if": "ctx.error?.message != null"
}
},
{
"pipeline": {
"name": "global@custom",
@@ -405,7 +459,14 @@
{
"append": {
"field": "error.message",
"value": "{{{ _ingest.on_failure_message }}}"
"value": "Processor '{{{ _ingest.on_failure_processor_type }}}' {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'"
}
},
{
"append": {
"field": "tags",
"value": "preserve_original_event",
"allow_duplicates": false
}
}
]
salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1-suricata → salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2-suricata
View File
salt/elasticsearch/files/log4j2.properties
+4
View File
@@ -45,3 +45,7 @@ appender.rolling_json.strategy.action.condition.nested_condition.age = 1D
rootLogger.level = info
rootLogger.appenderRef.rolling.ref = rolling
rootLogger.appenderRef.rolling_json.ref = rolling_json
# Suppress NotEntitledException WARNs (ES 9.3.3 bug)
logger.entitlement_security.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.x-pack-security.org.elasticsearch.security.org.elasticsearch.xpack.security
logger.entitlement_security.level = error
salt/elasticsearch/template.map.jinja
+62 -18
View File
@@ -14,15 +14,42 @@
{% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}
{% set ALL_ADDON_INTEGRATION_DEFAULTS = {} %}
{% set ALL_ADDON_SETTINGS_ORIG = {} %}
{% set ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES = {} %}
{% set ALL_ADDON_SETTINGS = {} %}
{# start generation of integration default index_settings #}
{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
{% set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
{% if check_package_components.size > 1 %}
{% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
{% for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %}
{% do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %}
{% endfor %}
{% endif%}
{% if salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
{# import integration type defaults #}
{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %}
{% set check_integration_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
{% if check_integration_package_components.size > 1 %}
{% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
{% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INTEGRATION_DEFAULTS) %}
{% endif %}
{% endif %}
{# import input type defaults #}
{% if salt['file.file_exists']('/opt/so/state/esfleet_input_package_components.json') %}
{% set check_input_package_components = salt['file.stats']('/opt/so/state/esfleet_input_package_components.json') %}
{% if check_input_package_components.size > 1 %}
{% from 'elasticfleet/input-defaults.map.jinja' import ADDON_INPUT_INTEGRATION_DEFAULTS %}
{% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INPUT_INTEGRATION_DEFAULTS) %}
{% endif %}
{% endif %}
{# import content type defaults #}
{% if salt['file.file_exists']('/opt/so/state/esfleet_content_package_components.json') %}
{% set check_content_package_components = salt['file.stats']('/opt/so/state/esfleet_content_package_components.json') %}
{% if check_content_package_components.size > 1 %}
{% from 'elasticfleet/content-defaults.map.jinja' import ADDON_CONTENT_INTEGRATION_DEFAULTS %}
{% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_CONTENT_INTEGRATION_DEFAULTS) %}
{% endif %}
{% endif %}
{% for index, settings in ALL_ADDON_INTEGRATION_DEFAULTS.items() %}
{% do ALL_ADDON_SETTINGS_ORIG.update({index: settings}) %}
{% endfor %}
{% endif %}
{# end generation of integration default index_settings #}
@@ -31,25 +58,33 @@
{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
{% endfor %}
{% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %}
{% for index in ALL_ADDON_SETTINGS_ORIG.keys() %}
{% do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
{% endfor %}
{% endif %}
{% set ES_INDEX_SETTINGS = {} %}
{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
{% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %}
{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS) %}
{% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
{% for index, settings in GLOBAL_OVERRIDES.items() %}
{# prevent this action from being performed on custom defined indices. #}
{# the custom defined index is not present in either of the dictionaries and fails to reder. #}
{% if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %}
{% if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %}
{# dont merge policy from the global_overrides if policy isn't defined in the original index settingss #}
{# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
{% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
{% if not DEFINED_SETTINGS[index].policy is defined and GLOBAL_OVERRIDES[index].policy is defined %}
{% do GLOBAL_OVERRIDES[index].pop('policy') %}
{% endif %}
{# this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #}
{% if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
{% for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %}
{% if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %}
{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
{% if GLOBAL_OVERRIDES[index].policy is defined %}
{% for phase in GLOBAL_OVERRIDES[index].policy.phases.copy() %}
{% if DEFINED_SETTINGS[index].policy.phases[phase] is not defined %}
{% do GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
{% endif %}
{% endfor %}
{% endif %}
@@ -111,5 +146,14 @@
{% endfor %}
{% endif %}
{% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %}
{% do FINAL_INDEX_SETTINGS.update({index | replace("_x_", "."): GLOBAL_OVERRIDES[index]}) %}
{% endfor %}
{% endmacro %}
{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS) }}
{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS) }}
{% set SO_MANAGED_INDICES = [] %}
{% for index, settings in ES_INDEX_SETTINGS.items() %}
{% do SO_MANAGED_INDICES.append(index) %}
{% endfor %}
salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list
+15 -4
View File
@@ -6,8 +6,19 @@
# Elastic License 2.0.
. /usr/sbin/so-common
if [ "$1" == "" ]; then
curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template | jq '.component_templates[] |.name'| sort
if [[ -z "$1" ]]; then
if output=$(so-elasticsearch-query "_component_template" --retry 3 --retry-delay 1 --fail); then
jq '[.component_templates[] | .name] | sort' <<< "$output"
else
echo "Failed to retrieve component templates from Elasticsearch."
exit 1
fi
else
curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template/$1 | jq
fi
if output=$(so-elasticsearch-query "_component_template/$1" --retry 3 --retry-delay 1 --fail); then
jq <<< "$output"
else
echo "Failed to retrieve component template '$1' from Elasticsearch."
exit 1
fi
fi
salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load
Executable
+253
View File
@@ -0,0 +1,253 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
SO_STATEFILE_SUCCESS=/opt/so/state/estemplates.txt
ADDON_STATEFILE_SUCCESS=/opt/so/state/addon_estemplates.txt
ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates"
SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index"
ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
SO_LOAD_FAILURES=0
ADDON_LOAD_FAILURES=0
SO_LOAD_FAILURES_NAMES=()
ADDON_LOAD_FAILURES_NAMES=()
IS_HEAVYNODE="false"
FORCE="false"
VERBOSE="false"
SHOULD_EXIT_ON_FAILURE="true"
# If soup is running, ignore errors
pgrep soup >/dev/null && SHOULD_EXIT_ON_FAILURE="false"
while [[ $# -gt 0 ]]; do
case "$1" in
--heavynode)
IS_HEAVYNODE="true"
;;
--force)
FORCE="true"
;;
--verbose)
VERBOSE="true"
;;
*)
echo "Usage: $0 [options]"
echo "Options:"
echo " --heavynode Only loads index templates specific to heavynodes"
echo " --force Force reload all templates regardless of statefiles (default: false)"
echo " --verbose Enable verbose output"
exit 1
;;
esac
shift
done
load_template() {
local uri="$1"
local file="$2"
echo "Loading template file $file"
if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then
echo "$output"
return 1
elif [[ "$VERBOSE" == "true" ]]; then
echo "$output"
fi
}
check_required_component_template_exists() {
local required
local missing
local file=$1
required=$(jq '[((.composed_of //[]) - (.ignore_missing_component_templates // []))[]]' "$file")
missing=$(jq -n --argjson required "$required" --argjson component_templates "$component_templates" '(($required) - ($component_templates))')
if [[ $(jq length <<<"$missing") -gt 0 ]]; then
return 1
fi
}
check_heavynode_compatiable_index_template() {
# The only templates that are relevant to heavynodes are from datasets defined in elasticagent/files/elastic-agent.yml.jinja.
# Heavynodes do not have fleet server packages installed and do not support elastic agents reporting directly to them.
local -A heavynode_index_templates=(
["so-import"]=1
["so-syslog"]=1
["so-logs-soc"]=1
["so-suricata"]=1
["so-suricata.alerts"]=1
["so-zeek"]=1
["so-strelka"]=1
)
local template_name="$1"
if [[ ! -v heavynode_index_templates["$template_name"] ]]; then
return 1
fi
}
load_component_templates() {
local printed_name="$1"
local pattern="${ELASTICSEARCH_TEMPLATES_DIR}/component/$2"
local append_mappings="${3:-"false"}"
# current state of nullglob shell option
shopt -q nullglob && nullglob_set=1 || nullglob_set=0
shopt -s nullglob
echo -e "\nLoading $printed_name component templates...\n"
for component in "$pattern"/*.json; do
tmpl_name=$(basename "${component%.json}")
if [[ "$append_mappings" == "true" ]]; then
# avoid duplicating "-mappings" if it already exists in the component template filename
tmpl_name="${tmpl_name%-mappings}-mappings"
fi
if ! load_template "_component_template/${tmpl_name}" "$component"; then
SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
SO_LOAD_FAILURES_NAMES+=("$component")
fi
done
# restore nullglob shell option if needed
if [[ $nullglob_set -eq 1 ]]; then
shopt -u nullglob
fi
}
check_elasticsearch_responsive() {
# Cannot load templates if Elasticsearch is not responding.
# NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the
# script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state.
retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
}
if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]]; then
check_elasticsearch_responsive
if [[ "$IS_HEAVYNODE" == "false" ]]; then
# TODO: Better way to check if fleet server is installed vs checking for Elastic Defend component template.
fleet_check="logs-endpoint.alerts@package"
if ! so-elasticsearch-query "_component_template/$fleet_check" --output /dev/null --retry 5 --retry-delay 3 --fail; then
# This check prevents so-elasticsearch-templates-load from running before so-elastic-fleet-setup has run.
echo -e "\nPackage $fleet_check not yet installed. Fleet Server may not be fully configured yet."
# Fleet Server is required because some SO index templates depend on components installed via
# specific integrations eg Elastic Defend. These are components that we do not manually create / manage
# via /opt/so/saltstack/salt/elasticsearch/templates/component/
exit 0
fi
fi
# load_component_templates "Name" "directory" "append '-mappings'?"
load_component_templates "ECS" "ecs" "true"
load_component_templates "Elastic Agent" "elastic-agent"
load_component_templates "Security Onion" "so"
component_templates=$(so-elasticsearch-component-templates-list)
echo -e "Loading Security Onion index templates...\n"
for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do
tmpl_name=$(basename "${so_idx_tmpl%-template.json}")
if [[ "$IS_HEAVYNODE" == "true" ]]; then
# TODO: Better way to load only heavynode specific templates
if ! check_heavynode_compatiable_index_template "$tmpl_name"; then
if [[ "$VERBOSE" == "true" ]]; then
echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template."
fi
continue
fi
fi
if check_required_component_template_exists "$so_idx_tmpl"; then
if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then
SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
fi
else
echo "Skipping over $so_idx_tmpl due to missing required component template(s)."
SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
continue
fi
done
if [[ $SO_LOAD_FAILURES -eq 0 ]]; then
echo "All Security Onion core templates loaded successfully."
touch "$SO_STATEFILE_SUCCESS"
else
echo "Encountered $SO_LOAD_FAILURES failure(s) loading templates:"
for failed_template in "${SO_LOAD_FAILURES_NAMES[@]}"; do
echo " - $failed_template"
done
if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
fail "Failed to load all Security Onion core templates successfully."
fi
fi
else
echo "Security Onion core templates already loaded"
fi
# Start loading addon templates
if [[ (-d "$ADDON_TEMPLATES_DIR" && -f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && ! -f "$ADDON_STATEFILE_SUCCESS") || (-d "$ADDON_TEMPLATES_DIR" && "$IS_HEAVYNODE" == "false" && "$FORCE" == "true") ]]; then
check_elasticsearch_responsive
echo -e "\nLoading addon integration index templates...\n"
component_templates=$(so-elasticsearch-component-templates-list)
for addon_idx_tmpl in "${ADDON_TEMPLATES_DIR}"/*.json; do
tmpl_name=$(basename "${addon_idx_tmpl%-template.json}")
if check_required_component_template_exists "$addon_idx_tmpl"; then
if ! load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl"; then
ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
fi
else
echo "Skipping over $addon_idx_tmpl due to missing required component template(s)."
ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
continue
fi
done
if [[ $ADDON_LOAD_FAILURES -eq 0 ]]; then
echo "All addon integration templates loaded successfully."
touch "$ADDON_STATEFILE_SUCCESS"
else
echo "Encountered $ADDON_LOAD_FAILURES failure(s) loading addon integration templates:"
for failed_template in "${ADDON_LOAD_FAILURES_NAMES[@]}"; do
echo " - $failed_template"
done
if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
fail "Failed to load all addon integration templates successfully."
fi
fi
elif [[ ! -f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" ]]; then
echo "Skipping loading addon integration templates until Security Onion core templates have been loaded."
elif [[ -f "$ADDON_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && "$FORCE" == "false" ]]; then
echo "Addon integration templates already loaded"
fi
salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load
+13
View File
@@ -7,6 +7,9 @@
. /usr/sbin/so-common
{%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
{%- if GLOBALS.role != "so-heavynode" %}
{%- from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
{%- endif %}
{%- for index, settings in ES_INDEX_SETTINGS.items() %}
{%- if settings.policy is defined %}
@@ -33,3 +36,13 @@
{%- endif %}
{%- endfor %}
echo
{%- if GLOBALS.role != "so-heavynode" %}
{%- for index, settings in ALL_ADDON_SETTINGS.items() %}
{%- if settings.policy is defined %}
echo
echo "Setting up {{ index }}-logs policy..."
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
echo
{%- endif %}
{%- endfor %}
{%- endif %}
salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load
-165
View File
@@ -1,165 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt
STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
if [[ -f $STATE_FILE_INITIAL ]]; then
# The initial template load has already run. As this is a subsequent load, all dependencies should
# already be satisified. Therefore, immediately exit/abort this script upon any template load failure
# since this is an unrecoverable failure.
should_exit_on_failure=1
else
# This is the initial template load, and there likely are some components not yet setup in Elasticsearch.
# Therefore load as many templates as possible at this time and if an error occurs proceed to the next
# template. But if at least one template fails to load do not mark the templates as having been loaded.
# This will allow the next load to resume the load of the templates that failed to load initially.
should_exit_on_failure=0
echo "This is the initial template load"
fi
# If soup is running, ignore errors
pgrep soup > /dev/null && should_exit_on_failure=0
load_failures=0
load_template() {
uri=$1
file=$2
echo "Loading template file $i"
if ! retry 3 1 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then
if [[ $should_exit_on_failure -eq 1 ]]; then
fail "Could not load template file: $file"
else
load_failures=$((load_failures+1))
echo "Incremented load failure counter: $load_failures"
fi
fi
}
if [ ! -f $STATE_FILE_SUCCESS ]; then
echo "State file $STATE_FILE_SUCCESS not found. Running so-elasticsearch-templates-load."
. /usr/sbin/so-common
{% if GLOBALS.role != 'so-heavynode' %}
if [ -f /usr/sbin/so-elastic-fleet-common ]; then
. /usr/sbin/so-elastic-fleet-common
fi
{% endif %}
default_conf_dir=/opt/so/conf
# Define a default directory to load pipelines from
ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
{% if GLOBALS.role == 'so-heavynode' %}
file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json"
{% else %}
file="/usr/sbin/so-elastic-fleet-common"
{% endif %}
if [ -f "$file" ]; then
# Wait for ElasticSearch to initialize
echo -n "Waiting for ElasticSearch..."
retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'"
{% if GLOBALS.role != 'so-heavynode' %}
TEMPLATE="logs-endpoint.alerts@package"
INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name)
if [ "$INSTALLED" != "$TEMPLATE" ]; then
echo
echo "Packages not yet installed."
echo
exit 0
fi
{% endif %}
touch $STATE_FILE_INITIAL
cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
echo "Loading ECS component templates..."
for i in *; do
TEMPLATE=$(echo $i | cut -d '.' -f1)
load_template "_component_template/${TEMPLATE}-mappings" "$i"
done
echo
cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent
echo "Loading Elastic Agent component templates..."
{% if GLOBALS.role == 'so-heavynode' %}
component_pattern="so-*"
{% else %}
component_pattern="*"
{% endif %}
for i in $component_pattern; do
TEMPLATE=${i::-5}
load_template "_component_template/$TEMPLATE" "$i"
done
echo
# Load SO-specific component templates
cd ${ELASTICSEARCH_TEMPLATES}/component/so
echo "Loading Security Onion component templates..."
for i in *; do
TEMPLATE=$(echo $i | cut -d '.' -f1);
load_template "_component_template/$TEMPLATE" "$i"
done
echo
# Load SO index templates
cd ${ELASTICSEARCH_TEMPLATES}/index
echo "Loading Security Onion index templates..."
shopt -s extglob
{% if GLOBALS.role == 'so-heavynode' %}
pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*|*endpoint*|*elasticsearch*|*generic*|*fleet_server*|*soc*)"
{% else %}
pattern="*"
{% endif %}
# Index templates will be skipped if the following conditions are met:
# 1. The template is part of the "so-logs-" template group
# 2. The template name does not correlate to at least one existing component template
# In this situation, the script will treat the skipped template as a temporary failure
# and allow the templates to be loaded again on the next run or highstate, whichever
# comes first.
COMPONENT_LIST=$(so-elasticsearch-component-templates-list)
for i in $pattern; do
TEMPLATE=${i::-14}
COMPONENT_PATTERN=${TEMPLATE:3}
MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ \.generic|logs-winlog\.winlog ]]; then
load_failures=$((load_failures+1))
echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
else
load_template "_index_template/$TEMPLATE" "$i"
fi
done
else
{% if GLOBALS.role == 'so-heavynode' %}
echo "Common template does not exist. Exiting..."
{% else %}
echo "Elastic Fleet not configured. Exiting..."
{% endif %}
exit 0
fi
cd - >/dev/null
if [[ $load_failures -eq 0 ]]; then
echo "All templates loaded successfully"
touch $STATE_FILE_SUCCESS
else
echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate"
fi
else
echo "Templates already loaded"
fi
salt/global/soc_global.yaml
-8
View File
@@ -11,18 +11,14 @@ global:
regexFailureMessage: You must enter a valid IP address or CIDR.
mdengine:
description: Which engine to use for meta data generation. Options are ZEEK and SURICATA.
regex: ^(ZEEK|SURICATA)$
options:
- ZEEK
- SURICATA
regexFailureMessage: You must enter either ZEEK or SURICATA.
global: True
pcapengine:
description: Which engine to use for generating pcap. Currently only SURICATA is supported.
regex: ^(SURICATA)$
options:
- SURICATA
regexFailureMessage: You must enter either SURICATA.
global: True
ids:
description: Which IDS engine to use. Currently only Suricata is supported.
@@ -42,11 +38,9 @@ global:
advanced: True
pipeline:
description: Sets which pipeline technology for events to use. The use of Kafka requires a Security Onion Pro license.
regex: ^(REDIS|KAFKA)$
options:
- REDIS
- KAFKA
regexFailureMessage: You must enter either REDIS or KAFKA.
global: True
advanced: True
repo_host:
@@ -67,12 +61,10 @@ global:
advanced: True
telegraf_output:
description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation.
regex: ^(INFLUXDB|POSTGRES|BOTH)$
options:
- INFLUXDB
- POSTGRES
- BOTH
regexFailureMessage: You must enter INFLUXDB, POSTGRES, or BOTH.
global: True
advanced: True
helpLink: influxdb
salt/influxdb/soc_influxdb.yaml
+10 -3
View File
@@ -85,7 +85,10 @@ influxdb:
description: The log level to use for outputting log statements. Allowed values are debug, info, or error.
global: True
advanced: false
regex: ^(info|debug|error)$
options:
- info
- debug
- error
helpLink: influxdb
metrics-disabled:
description: If true, the HTTP endpoint that exposes internal InfluxDB metrics will be inaccessible.
@@ -140,7 +143,9 @@ influxdb:
description: Determines the type of storage used for secrets. Allowed values are bolt or vault.
global: True
advanced: True
regex: ^(bolt|vault)$
options:
- bolt
- vault
helpLink: influxdb
session-length:
description: Number of minutes that a user login session can remain authenticated.
@@ -260,7 +265,9 @@ influxdb:
description: The type of data store to use for HTTP resources. Allowed values are disk or memory. Memory should not be used for production Security Onion installations.
global: True
advanced: True
regex: ^(disk|memory)$
options:
- disk
- memory
helpLink: influxdb
tls-cert:
description: The container path to the certificate to use for TLS encryption of the HTTP requests and responses.
salt/kafka/soc_kafka.yaml
+14 -4
View File
@@ -128,10 +128,13 @@ kafka:
title: ssl.keystore.password
sensitive: True
helpLink: kafka
ssl_x_keystore_x_type:
ssl_x_keystore_x_type:
description: The key store file format.
title: ssl.keystore.type
regex: ^(JKS|PKCS12|PEM)$
options:
- JKS
- PKCS12
- PEM
helpLink: kafka
ssl_x_truststore_x_location:
description: The trust store file location within the Docker container.
@@ -160,7 +163,11 @@ kafka:
security_x_protocol:
description: 'Broker communication protocol. Options are: SASL_SSL, PLAINTEXT, SSL, SASL_PLAINTEXT'
title: security.protocol
regex: ^(SASL_SSL|PLAINTEXT|SSL|SASL_PLAINTEXT)
options:
- SASL_SSL
- PLAINTEXT
- SSL
- SASL_PLAINTEXT
helpLink: kafka
ssl_x_keystore_x_location:
description: The key store file location within the Docker container.
@@ -174,7 +181,10 @@ kafka:
ssl_x_keystore_x_type:
description: The key store file format.
title: ssl.keystore.type
regex: ^(JKS|PKCS12|PEM)$
options:
- JKS
- PKCS12
- PEM
helpLink: kafka
ssl_x_truststore_x_location:
description: The trust store file location within the Docker container.
salt/kibana/tools/sbin_jinja/so-kibana-space-defaults
+1 -1
View File
@@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http:
# Disable certain Features from showing up in the Kibana UI
echo
echo "Setting up default Kibana Space:"
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","entityManager"]} ' >> /opt/so/log/kibana/misc.log
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log
echo
salt/kratos/soc_kratos.yaml
+9 -4
View File
@@ -21,8 +21,12 @@ kratos:
description: "Specify the provider type. Required. Valid values are: auth0, generic, github, google, microsoft"
global: True
forcedType: string
regex: "auth0|generic|github|google|microsoft"
regexFailureMessage: "Valid values are: auth0, generic, github, google, microsoft"
options:
- auth0
- generic
- github
- google
- microsoft
helpLink: oidc
client_id:
description: Specify the client ID, also referenced as the application ID. Required.
@@ -43,8 +47,9 @@ kratos:
description: The source of the subject identifier. Typically 'userinfo'. Only used when provider is 'microsoft'.
global: True
forcedType: string
regex: me|userinfo
regexFailureMessage: "Valid values are: me, userinfo"
options:
- me
- userinfo
helpLink: oidc
auth_url:
description: Provider's auth URL. Required when provider is 'generic'.
salt/manager/tools/sbin/so-minion
+2 -2
View File
@@ -132,8 +132,8 @@ function getinstallinfo() {
log "ERROR" "Failed to get install info from $MINION_ID"
return 1
fi
export $(echo "$INSTALLVARS" | xargs)
while read -r var; do export "$var"; done <<< "$INSTALLVARS"
if [ $? -ne 0 ]; then
log "ERROR" "Failed to source install variables"
return 1
salt/manager/tools/sbin/soup
+6 -5
View File
@@ -363,7 +363,8 @@ preupgrade_changes() {
echo "Checking to see if changes are needed."
[[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0
[[ "$INSTALLEDVERSION" =~ ^3\.0\.[0-9]+$ ]] && up_to_3.1.0
[[ "$INSTALLEDVERSION" == 3.0.0 ]] && up_to_3.1.0
true
}
@@ -372,7 +373,8 @@ postupgrade_changes() {
echo "Running post upgrade processes."
[[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
[[ "$POSTVERSION" =~ ^3\.0\.[0-9]+$ ]] && post_to_3.1.0
[[ "$POSTVERSION" =~ 3.0.0 ]] && post_to_3.1.0
true
}
@@ -383,7 +385,7 @@ check_minimum_version() {
fi
}
### 3.0.0 Scripts ###
### 3.0.0 Start ###
convert_suricata_yes_no() {
echo "Starting suricata yes/no values to true/false conversion."
@@ -472,8 +474,8 @@ post_to_3.0.0() {
### 3.0.0 End ###
### 3.1.0 Start ###
up_to_3.1.0() {
echo "Nothing to do"
INSTALLEDVERSION=3.1.0
}
@@ -485,7 +487,6 @@ post_to_3.1.0() {
# picks up the new credentials on the first apply after soup.
echo "Provisioning Telegraf Postgres users for existing minions."
salt-call --local state.apply postgres.auth postgres.telegraf_users || true
salt '*' state.sls telegraf || true
POSTVERSION=3.1.0
}
salt/postgres/defaults.yaml
+2
View File
@@ -14,3 +14,5 @@ postgres:
log_destination: 'stderr'
logging_collector: 'off'
log_min_messages: 'warning'
shared_preload_libraries: pg_cron
cron.database_name: so_telegraf
salt/postgres/enabled.sls
+3 -9
View File
@@ -80,20 +80,14 @@ delete_so-postgres_so-status.disabled:
- name: /opt/so/conf/so-status/so-status.conf
- regex: ^so-postgres$
# Retention is now handled by pg_partman (hourly maintenance via pg_cron
# scheduled from postgres/telegraf_users.sls). The so-telegraf-trim script
# stays on disk for manual/emergency use but is no longer scheduled.
so_telegraf_trim:
{% if GLOBALS.telegraf_output in ['POSTGRES', 'BOTH'] %}
cron.present:
{% else %}
cron.absent:
{% endif %}
- name: /usr/sbin/so-telegraf-trim >> /opt/so/log/postgres/telegraf-trim.log 2>&1
- identifier: so_telegraf_trim
- user: root
- minute: '17'
- hour: '3'
- daymonth: '*'
- month: '*'
- dayweek: '*'
{% else %}
salt/postgres/telegraf_users.sls
+54 -2
View File
@@ -10,6 +10,36 @@
{% set TG_OUT = (GLOBALS.telegraf_output | default('INFLUXDB')) | upper %}
{% if TG_OUT in ['POSTGRES', 'BOTH'] %}
# Provision the shared group role and schema once. Every per-minion role is a
# member of so_telegraf, and each Telegraf connection does SET ROLE so_telegraf
# (via options='-c role=so_telegraf' in the connection string) so tables created
# on first write are owned by the group role and every member can INSERT/SELECT.
postgres_telegraf_group_role:
cmd.run:
- name: |
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
DO $$
BEGIN
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_telegraf') THEN
CREATE ROLE so_telegraf NOLOGIN;
END IF;
END
$$;
GRANT CONNECT ON DATABASE so_telegraf TO so_telegraf;
CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf;
GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf;
CREATE EXTENSION IF NOT EXISTS pg_partman;
CREATE EXTENSION IF NOT EXISTS pg_cron;
-- Hourly partman maintenance. cron.schedule is idempotent by jobname.
SELECT cron.schedule(
'telegraf-partman-maintenance',
'17 * * * *',
'CALL partman.run_maintenance_proc()'
);
EOSQL
- require:
- docker_container: so-postgres
{% set users = salt['pillar.get']('postgres:auth:users', {}) %}
{% for key, entry in users.items() %}
{% if key.startswith('telegraf_') and entry.get('user') and entry.get('pass') %}
@@ -30,14 +60,36 @@ postgres_telegraf_role_{{ u }}:
END
$$;
GRANT CONNECT ON DATABASE so_telegraf TO "{{ u }}";
CREATE SCHEMA IF NOT EXISTS "{{ u }}" AUTHORIZATION "{{ u }}";
GRANT so_telegraf TO "{{ u }}";
EOSQL
- require:
- docker_container: so-postgres
- cmd: postgres_telegraf_group_role
{% endif %}
{% endfor %}
# Reconcile partman retention from pillar. Runs after role/schema setup so
# any partitioned parents Telegraf has already created get their retention
# refreshed whenever postgres.telegraf.retention_days changes.
{% set retention = salt['pillar.get']('postgres:telegraf:retention_days', 14) %}
postgres_telegraf_retention_reconcile:
cmd.run:
- name: |
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman') THEN
UPDATE partman.part_config
SET retention = '{{ retention }} days',
retention_keep_table = false
WHERE parent_table LIKE 'telegraf.%';
END IF;
END
$$;
EOSQL
- require:
- cmd: postgres_telegraf_group_role
{% endif %}
{% else %}
salt/postgres/tools/sbin/so-show-stats
-145
View File
@@ -1,145 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
# Point-in-time host metrics from the Telegraf Postgres backend.
# Sanity-check tool for verifying metrics are landing before the grid
# dashboards consume them.
. /usr/sbin/so-common
usage() {
cat <<EOF
Usage: $0 [minion_id]
Shows the most recent CPU, memory, disk, and load metrics for each minion
from the so_telegraf Postgres database. Without an argument, reports on
every minion that has data. With a minion_id, limits output to that one.
Requires: sudo, so-postgres running, global.telegraf_output set to
POSTGRES or BOTH.
EOF
exit 1
}
if [ "$(id -u)" -ne 0 ]; then
echo "This script must be run using sudo!"
exit 1
fi
case "${1:-}" in
-h|--help) usage ;;
esac
FILTER_MINION="${1:-}"
so_psql() {
docker exec so-postgres psql -U postgres -d so_telegraf -At -F $'\t' "$@"
}
if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 | grep -qw so_telegraf; then
echo "Database so_telegraf not found. Is global.telegraf_output set to POSTGRES or BOTH?"
exit 2
fi
# List telegraf schemas (role-per-minion naming convention: so_telegraf_<sanitized_minion_id>)
SCHEMAS=$(so_psql -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name LIKE 'so_telegraf_%' ORDER BY schema_name;")
if [ -z "$SCHEMAS" ]; then
echo "No minion schemas found in so_telegraf."
exit 0
fi
print_metric() {
local schema="$1" table="$2" query="$3"
# Confirm table exists in this schema before querying
local exists
exists=$(so_psql -c "SELECT 1 FROM information_schema.tables WHERE table_schema='${schema}' AND table_name='${table}' LIMIT 1;")
[ -z "$exists" ] && return 0
so_psql -c "$query"
}
# Telegraf's postgresql output stores tag values either as individual columns
# on the <metric>_tag table or as a single JSONB "tags" column, depending on
# plugin version. Returns a SQL expression that extracts the named tag
# regardless of layout. Empty string if the tag table doesn't exist.
tag_expr() {
local schema="$1" table="$2" tag="$3" alias="$4"
local has_col
has_col=$(so_psql -c "
SELECT 1 FROM information_schema.columns
WHERE table_schema='${schema}' AND table_name='${table}_tag' AND column_name='${tag}'
LIMIT 1;")
if [ -n "$has_col" ]; then
echo "${alias}.${tag}"
return
fi
local has_tags
has_tags=$(so_psql -c "
SELECT 1 FROM information_schema.columns
WHERE table_schema='${schema}' AND table_name='${table}_tag' AND column_name='tags'
LIMIT 1;")
if [ -n "$has_tags" ]; then
echo "(${alias}.tags->>'${tag}')"
return
fi
echo ""
}
for schema in $SCHEMAS; do
minion="${schema#so_telegraf_}"
if [ -n "$FILTER_MINION" ]; then
# Compare against the sanitized form used in schema names
want=$(echo "$FILTER_MINION" | tr '.-' '_' | tr '[:upper:]' '[:lower:]')
[ "$minion" != "$want" ] && continue
fi
echo "===================================================================="
echo " Minion: $minion"
echo "===================================================================="
cpu_tag=$(tag_expr "$schema" "cpu" "cpu" "t")
if [ -n "$cpu_tag" ]; then
print_metric "$schema" "cpu" "
SELECT 'cpu ' AS metric,
to_char(c.time, 'YYYY-MM-DD HH24:MI:SS') AS ts,
round((100 - c.usage_idle)::numeric, 1) || '% used'
FROM \"${schema}\".cpu c
JOIN \"${schema}\".cpu_tag t USING (tag_id)
WHERE ${cpu_tag} = 'cpu-total'
ORDER BY c.time DESC LIMIT 1;"
fi
print_metric "$schema" "mem" "
SELECT 'memory ' AS metric,
to_char(m.time, 'YYYY-MM-DD HH24:MI:SS') AS ts,
round(m.used_percent::numeric, 1) || '% used (' ||
pg_size_pretty(m.used) || ' of ' || pg_size_pretty(m.total) || ')'
FROM \"${schema}\".mem m
ORDER BY m.time DESC LIMIT 1;"
disk_path=$(tag_expr "$schema" "disk" "path" "t")
if [ -n "$disk_path" ]; then
print_metric "$schema" "disk" "
SELECT 'disk ' || rpad(${disk_path}, 12) AS metric,
to_char(d.time, 'YYYY-MM-DD HH24:MI:SS') AS ts,
round(d.used_percent::numeric, 1) || '% used (' ||
pg_size_pretty(d.used) || ' of ' || pg_size_pretty(d.total) || ')'
FROM \"${schema}\".disk d
JOIN \"${schema}\".disk_tag t USING (tag_id)
WHERE d.time = (SELECT max(time) FROM \"${schema}\".disk)
ORDER BY ${disk_path};"
fi
print_metric "$schema" "system" "
SELECT 'load ' AS metric,
to_char(s.time, 'YYYY-MM-DD HH24:MI:SS') AS ts,
s.load1 || ' / ' || s.load5 || ' / ' || s.load15 || ' (1/5/15m)'
FROM \"${schema}\".system s
ORDER BY s.time DESC LIMIT 1;"
echo ""
done
salt/postgres/tools/sbin/so-stats-show
+144
View File
@@ -0,0 +1,144 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
# Point-in-time host metrics from the Telegraf Postgres backend.
# Sanity-check tool for verifying metrics are landing before the grid
# dashboards consume them.
#
# Assumes Telegraf's postgresql output is configured with
# tags_as_foreign_keys = true, tags_as_jsonb = true, fields_as_jsonb = true,
# so metric tables are (time, tag_id, fields jsonb) and tag tables are
# (tag_id, tags jsonb).
. /usr/sbin/so-common
usage() {
cat <<EOF
Usage: $0 [host]
Shows the most recent CPU, memory, disk, and load metrics for each host
from the so_telegraf Postgres database. Without an argument, reports on
every host that has data. With a host, limits output to that one.
Requires: sudo, so-postgres running, global.telegraf_output set to
POSTGRES or BOTH.
EOF
exit 1
}
if [ "$(id -u)" -ne 0 ]; then
echo "This script must be run using sudo!"
exit 1
fi
case "${1:-}" in
-h|--help) usage ;;
esac
FILTER_HOST="${1:-}"
SCHEMA="telegraf"
so_psql() {
docker exec so-postgres psql -U postgres -d so_telegraf -At -F $'\t' "$@"
}
if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 | grep -qw so_telegraf; then
echo "Database so_telegraf not found. Is global.telegraf_output set to POSTGRES or BOTH?"
exit 2
fi
table_exists() {
local table="$1"
[ -n "$(so_psql -c "SELECT 1 FROM information_schema.tables WHERE table_schema='${SCHEMA}' AND table_name='${table}' LIMIT 1;")" ]
}
# Discover hosts from cpu_tag (every minion reports cpu).
if ! table_exists "cpu_tag"; then
echo "${SCHEMA}.cpu_tag not found. Has Telegraf written any rows yet?"
exit 0
fi
HOSTS=$(so_psql -c "
SELECT DISTINCT tags->>'host'
FROM \"${SCHEMA}\".cpu_tag
WHERE tags ? 'host'
ORDER BY 1;")
if [ -z "$HOSTS" ]; then
echo "No hosts found in ${SCHEMA}. Is Telegraf configured to write to Postgres?"
exit 0
fi
print_metric() {
so_psql -c "$1"
}
for host in $HOSTS; do
if [ -n "$FILTER_HOST" ] && [ "$host" != "$FILTER_HOST" ]; then
continue
fi
echo "===================================================================="
echo " Host: $host"
echo "===================================================================="
if table_exists "cpu"; then
print_metric "
SELECT 'cpu ' AS metric,
to_char(c.time, 'YYYY-MM-DD HH24:MI:SS') AS ts,
round((100 - (c.fields->>'usage_idle')::numeric), 1) || '% used'
FROM \"${SCHEMA}\".cpu c
JOIN \"${SCHEMA}\".cpu_tag t USING (tag_id)
WHERE t.tags->>'host' = '${host}' AND t.tags->>'cpu' = 'cpu-total'
ORDER BY c.time DESC LIMIT 1;"
fi
if table_exists "mem"; then
print_metric "
SELECT 'memory ' AS metric,
to_char(m.time, 'YYYY-MM-DD HH24:MI:SS') AS ts,
round((m.fields->>'used_percent')::numeric, 1) || '% used (' ||
pg_size_pretty((m.fields->>'used')::bigint) || ' of ' ||
pg_size_pretty((m.fields->>'total')::bigint) || ')'
FROM \"${SCHEMA}\".mem m
JOIN \"${SCHEMA}\".mem_tag t USING (tag_id)
WHERE t.tags->>'host' = '${host}'
ORDER BY m.time DESC LIMIT 1;"
fi
if table_exists "disk"; then
print_metric "
SELECT 'disk ' || rpad(t.tags->>'path', 12) AS metric,
to_char(d.time, 'YYYY-MM-DD HH24:MI:SS') AS ts,
round((d.fields->>'used_percent')::numeric, 1) || '% used (' ||
pg_size_pretty((d.fields->>'used')::bigint) || ' of ' ||
pg_size_pretty((d.fields->>'total')::bigint) || ')'
FROM \"${SCHEMA}\".disk d
JOIN \"${SCHEMA}\".disk_tag t USING (tag_id)
WHERE t.tags->>'host' = '${host}'
AND d.time = (SELECT max(d2.time)
FROM \"${SCHEMA}\".disk d2
JOIN \"${SCHEMA}\".disk_tag t2 USING (tag_id)
WHERE t2.tags->>'host' = '${host}')
ORDER BY t.tags->>'path';"
fi
if table_exists "system"; then
print_metric "
SELECT 'load ' AS metric,
to_char(s.time, 'YYYY-MM-DD HH24:MI:SS') AS ts,
(s.fields->>'load1') || ' / ' ||
(s.fields->>'load5') || ' / ' ||
(s.fields->>'load15') || ' (1/5/15m)'
FROM \"${SCHEMA}\".system s
JOIN \"${SCHEMA}\".system_tag t USING (tag_id)
WHERE t.tags->>'host' = '${host}'
ORDER BY s.time DESC LIMIT 1;"
fi
echo ""
done
salt/postgres/tools/sbin/so-telegraf-trim
+4 -4
View File
@@ -63,15 +63,15 @@ log "Trimming rows older than ${DAYS} days (dry_run=${DRY_RUN})."
TOTAL_DELETED=0
# One row per (schema, table) we might want to trim.
# Column name is 'time' for all telegraf output plugin tables; skip metadata
# tables (tag_* used for tags_as_foreign_keys).
# Every metric table in the shared telegraf schema has a 'time' column.
# Tag tables (<metric>_tag) don't, so filtering on the column presence is
# enough to scope the trim to metric tables only.
ROWS=$(so_psql -c "
SELECT table_schema || '.' || table_name
FROM information_schema.columns
WHERE column_name = 'time'
AND data_type IN ('timestamp with time zone', 'timestamp without time zone')
AND table_schema LIKE 'so_telegraf_%'
AND table_schema = 'telegraf'
ORDER BY 1;")
if [ -z "$ROWS" ]; then
salt/soc/defaults.yaml
+1
View File
@@ -2687,4 +2687,5 @@ soc:
lowBalanceColorAlert: 500000
enabled: true
adapter: SOAI
charsPerTokenEstimate: 4
salt/soc/soc_soc.yaml
+5 -1
View File
@@ -761,7 +761,7 @@ soc:
required: True
- field: origin
label: Country of Origin for the Model Training
required: false
required: False
- field: contextLimitSmall
label: Context Limit (Small)
forcedType: int
@@ -779,6 +779,10 @@ soc:
- field: enabled
label: Enabled
forcedType: bool
- field: charsPerTokenEstimate
label: Characters per Token Estimate
forcedType: float
required: False
apiTimeoutMs:
description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI.
global: True
salt/suricata/map.jinja
+1 -1
View File
@@ -33,7 +33,7 @@
{% do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %}
{% do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %}
{# multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #}
{% set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round | int %}
{% set maxfiles = ([1, (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round(0, 'ceil') | int] | max) %}
{% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %}
{% endif %}
salt/suricata/soc_suricata.yaml
+7 -3
View File
@@ -64,8 +64,10 @@ suricata:
helpLink: suricata
conditional:
description: Set to "all" to record PCAP for all flows. Set to "alerts" to only record PCAP for Suricata alerts. Set to "tag" to only record PCAP for tagged rules.
regex: ^(all|alerts|tag)$
regexFailureMessage: You must enter either all, alert or tag.
options:
- all
- alerts
- tag
helpLink: suricata
dir:
description: Parent directory to store PCAP.
@@ -83,7 +85,9 @@ suricata:
advanced: True
cluster-type:
advanced: True
regex: ^(cluster_flow|cluster_qm)$
options:
- cluster_flow
- cluster_qm
defrag:
description: Enable defragmentation of IP packets before processing.
forcedType: bool
salt/telegraf/etc/telegraf.conf
+17 -3
View File
@@ -94,11 +94,25 @@
{%- endif %}
{%- if TG_OUT in ['POSTGRES', 'BOTH'] %}
# Configuration for sending metrics to PostgreSQL
# Configuration for sending metrics to PostgreSQL.
# options='-c role=so_telegraf' makes every connection SET ROLE to the shared
# group role so tables created on first write are owned by so_telegraf, and
# all per-minion members can INSERT/SELECT them via role inheritance.
# fields_as_jsonb/tags_as_jsonb keep metric tables at a fixed column count so
# high-cardinality inputs (docker, procstat, kafka) don't blow past the
# Postgres 1600-column-per-table limit.
[[outputs.postgresql]]
connection = "host={{ PG_HOST }} port=5432 user={{ PG_USER }} password={{ PG_PASS }} dbname=so_telegraf sslmode=verify-full sslrootcert=/etc/telegraf/ca.crt"
schema = "{{ PG_USER }}"
connection = "host={{ PG_HOST }} port=5432 user={{ PG_USER }} password={{ PG_PASS }} dbname=so_telegraf sslmode=verify-full sslrootcert=/etc/telegraf/ca.crt options='-c role=so_telegraf'"
schema = "telegraf"
tags_as_foreign_keys = true
tags_as_jsonb = true
fields_as_jsonb = true
# Every metric table is a daily time-range partitioned parent managed by
# pg_partman. Retention drops old partitions instead of row-by-row DELETEs.
create_templates = [
'''CREATE TABLE {TABLE} ({COLUMNS}) PARTITION BY RANGE ("time")''',
'''SELECT partman.create_parent(p_parent_table := {TABLELITERAL}, p_control := 'time', p_type := 'native', p_interval := '1 day', p_premake := 3)'''
]
{%- endif %}
###############################################################################
setup/so-setup
+1
View File
@@ -219,6 +219,7 @@ if [ -n "$test_profile" ]; then
WEBUSER=onionuser@somewhere.invalid
WEBPASSWD1=0n10nus3r
WEBPASSWD2=0n10nus3r
NODE_DESCRIPTION="${HOSTNAME} - ${install_type} - ${MAINIP}"
update_sudoers_for_testing
fi