diff --git a/pillar/elasticsearch/index_templates.sls b/pillar/elasticsearch/index_templates.sls deleted file mode 100644 index a02a1818c..000000000 --- a/pillar/elasticsearch/index_templates.sls +++ /dev/null @@ -1,2 +0,0 @@ -elasticsearch: - index_settings: diff --git a/pillar/top.sls b/pillar/top.sls index af18bee09..808182c2b 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -102,7 +102,6 @@ base: - node_data.ips - secrets - healthcheck.eval - - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} @@ -152,7 +151,6 @@ base: - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} @@ -271,7 +269,6 @@ base: '*_import': - node_data.ips - secrets - - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} diff --git a/salt/elasticfleet/content-defaults.map.jinja b/salt/elasticfleet/content-defaults.map.jinja new file mode 100644 index 000000000..f4237d6d1 --- /dev/null +++ b/salt/elasticfleet/content-defaults.map.jinja @@ -0,0 +1,123 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use + this file except in compliance with the Elastic License 2.0. #} + + +{% import_json '/opt/so/state/esfleet_content_package_components.json' as ADDON_CONTENT_PACKAGE_COMPONENTS %} +{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %} +{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} + +{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} +{% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %} +{% set DEBUG_STUFF = {} %} + +{% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %} +{% if pkg.name in CORE_ESFLEET_PACKAGES %} +{# skip core content packages #} +{% elif pkg.name not in CORE_ESFLEET_PACKAGES %} +{# generate defaults for each content package #} +{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0%} +{% for pattern in pkg.dataStreams %} +{# in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config) + title: generic is an artifact of that and is not in use #} +{% if pattern.title == "generic" %} +{% continue %} +{% endif %} +{% if "metrics-" in pattern.name %} +{% set integration_type = "metrics-" %} +{% elif "logs-" in pattern.name %} +{% set integration_type = "logs-" %} +{% else %} +{% set integration_type = "" %} +{% endif %} +{# on content integrations the component name is user defined at the time it is added to an agent policy #} +{% set component_name = pattern.title %} +{% set index_pattern = pattern.name %} +{# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #} +{% set component_name_x = component_name.replace(".","_x_") %} +{# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #} +{% set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %} +{# Default integration settings #} +{% set integration_defaults = { + "index_sorting": false, + "index_template": { + "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], + "index_patterns": [index_pattern], + "priority": 501, + "template": { + "settings": { + "index": { + "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, + "number_of_replicas": 0 + } + } + } + }, + "policy": { + "phases": { + "cold": { + "actions": { + "allocate":{ + "number_of_replicas": "" + }, + "set_priority": {"priority": 0} + }, + "min_age": "60d" + }, + "delete": { + "actions": { + "delete": {} + }, + "min_age": "365d" + }, + "hot": { + "actions": { + "rollover": { + "max_age": "30d", + "max_primary_shard_size": "50gb" + }, + "forcemerge":{ + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 100} + }, + "min_age": "0ms" + }, + "warm": { + "actions": { + "allocate": { + "number_of_replicas": "" + }, + "forcemerge": { + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 50} + }, + "min_age": "30d" + } + } + } + } %} + + +{% do ADDON_CONTENT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %} +{% endfor %} +{% else %} +{% endif %} +{% endif %} +{% endfor %} diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index a3132d3f4..022600083 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -1,5 +1,6 @@ elasticfleet: enabled: False + patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release. enable_manager_output: True config: server: diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json index ac03f3c1d..c1fd7f147 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json @@ -9,16 +9,22 @@ "namespace": "so", "description": "Zeek Import logs", "policy_id": "so-grid-nodes_general", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/zeek/logs/*.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "import", "pipeline": "", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -34,7 +40,8 @@ "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json index 545588521..83d153439 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json @@ -15,19 +15,25 @@ "version": "" }, "name": "kratos-logs", + "namespace": "so", "description": "Kratos logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "kratos", "pipeline": "kratos", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -48,10 +54,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json index 4af2b2921..9797b9e75 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json @@ -9,16 +9,22 @@ "namespace": "so", "description": "Zeek logs", "policy_id": "so-grid-nodes_general", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/zeek/logs/current/*.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "zeek", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"], @@ -30,10 +36,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index debfc73a3..c27da26f7 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "9.0.2", + "version": "9.3.0", "requires_root": true }, "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json b/salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json index 0be40a3d3..3eec63d26 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json @@ -6,21 +6,23 @@ "name": "agent-monitor", "namespace": "", "description": "", + "policy_id": "so-grid-nodes_general", "policy_ids": [ "so-grid-nodes_general" ], - "output_id": null, "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/agents/agent-monitor.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "agentmonitor", "pipeline": "elasticagent.monitor", "parsers": "", @@ -34,15 +36,16 @@ "ignore_older": "72h", "clean_inactive": -1, "harvester_limit": 0, - "fingerprint": true, + "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": 64, - "file_identity_native": false, + "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json index a4f944ba5..5dcd3012d 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "hydra-logs", + "namespace": "so", "description": "Hydra logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/hydra/hydra.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "hydra", "pipeline": "hydra", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -34,10 +40,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json index fef9c57fb..afaf77f0c 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "idh-logs", + "namespace": "so", "description": "IDH integration", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/idh/opencanary.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "idh", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -31,10 +37,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 50ffd5dc7..32d210172 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -4,26 +4,32 @@ "version": "" }, "name": "import-evtx-logs", + "namespace": "so", "description": "Import Windows EVTX logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/evtx/*.json" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "import", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "exclude_files": [ "\\.gz$" ], "include_files": [], - "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.6.1\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.6.1\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.6.1\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n target: data_stream\n fields:\n dataset: import", + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.15.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.15.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.15.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", "tags": [ "import" ], @@ -33,10 +39,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json index b8f3b0b29..3148b38e8 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "import-suricata-logs", + "namespace": "so", "description": "Import Suricata logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/suricata/eve*.json" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "import", "pipeline": "suricata.common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -32,10 +38,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json index 70259c3cf..f807c3b70 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json @@ -4,14 +4,18 @@ "version": "" }, "name": "rita-logs", + "namespace": "so", "description": "RITA Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ @@ -19,6 +23,8 @@ "/nsm/rita/exploded-dns.csv", "/nsm/rita/long-connections.csv" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "rita", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "exclude_files": [ @@ -33,10 +39,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json b/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json index a14e63559..24ed188f2 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json @@ -4,19 +4,25 @@ "version": "" }, "name": "so-ip-mappings", + "namespace": "so", "description": "IP Description mappings", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/custom-mappings/ip-descriptions.csv" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "hostnamemappings", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "exclude_files": [ @@ -32,10 +38,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json index f4fd38e9d..c04b738d3 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "soc-auth-sync-logs", + "namespace": "so", "description": "Security Onion - Elastic Auth Sync - Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/sync.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "soc", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -31,10 +37,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json index f1bdbc922..9d7812e42 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json @@ -4,20 +4,26 @@ "version": "" }, "name": "soc-detections-logs", + "namespace": "so", "description": "Security Onion Console - Detections Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/detections_runtime-status_sigma.log", "/opt/so/log/soc/detections_runtime-status_yara.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "soc", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -35,10 +41,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json index cb08d5b12..d1fa8b630 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "soc-salt-relay-logs", + "namespace": "so", "description": "Security Onion - Salt Relay - Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/salt-relay.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "soc", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -33,10 +39,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json index 11e686c3d..467544c9d 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "soc-sensoroni-logs", + "namespace": "so", "description": "Security Onion - Sensoroni - Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/sensoroni/sensoroni.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "soc", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -31,10 +37,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json index decb6b22a..37eb02ab1 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "soc-server-logs", + "namespace": "so", "description": "Security Onion Console Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/sensoroni-server.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "soc", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -33,10 +39,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json index 1f0203a91..3091baf44 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "strelka-logs", + "namespace": "so", "description": "Strelka Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "strelka", "pipeline": "strelka.file", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -31,10 +37,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json index 26dae5225..bb5cfd2c3 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "suricata-logs", + "namespace": "so", "description": "Suricata integration", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/suricata/eve*.json" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "suricata", "pipeline": "suricata.common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -31,10 +37,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/input-defaults.map.jinja b/salt/elasticfleet/input-defaults.map.jinja new file mode 100644 index 000000000..a02844330 --- /dev/null +++ b/salt/elasticfleet/input-defaults.map.jinja @@ -0,0 +1,123 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use + this file except in compliance with the Elastic License 2.0. #} + + +{% import_json '/opt/so/state/esfleet_input_package_components.json' as ADDON_INPUT_PACKAGE_COMPONENTS %} +{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %} +{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} + +{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} +{% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %} +{% set DEBUG_STUFF = {} %} + +{% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %} +{% if pkg.name in CORE_ESFLEET_PACKAGES %} +{# skip core input packages #} +{% elif pkg.name not in CORE_ESFLEET_PACKAGES %} +{# generate defaults for each input package #} +{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %} +{% for pattern in pkg.dataStreams %} +{# in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config) + title: generic is an artifact of that and is not in use #} +{% if pattern.title == "generic" %} +{% continue %} +{% endif %} +{% if "metrics-" in pattern.name %} +{% set integration_type = "metrics-" %} +{% elif "logs-" in pattern.name %} +{% set integration_type = "logs-" %} +{% else %} +{% set integration_type = "" %} +{% endif %} +{# on input integrations the component name is user defined at the time it is added to an agent policy #} +{% set component_name = pattern.title %} +{% set index_pattern = pattern.name %} +{# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #} +{% set component_name_x = component_name.replace(".","_x_") %} +{# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #} +{% set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %} +{# Default integration settings #} +{% set integration_defaults = { + "index_sorting": false, + "index_template": { + "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], + "index_patterns": [index_pattern], + "priority": 501, + "template": { + "settings": { + "index": { + "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, + "number_of_replicas": 0 + } + } + } + }, + "policy": { + "phases": { + "cold": { + "actions": { + "allocate":{ + "number_of_replicas": "" + }, + "set_priority": {"priority": 0} + }, + "min_age": "60d" + }, + "delete": { + "actions": { + "delete": {} + }, + "min_age": "365d" + }, + "hot": { + "actions": { + "rollover": { + "max_age": "30d", + "max_primary_shard_size": "50gb" + }, + "forcemerge":{ + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 100} + }, + "min_age": "0ms" + }, + "warm": { + "actions": { + "allocate": { + "number_of_replicas": "" + }, + "forcemerge": { + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 50} + }, + "min_age": "30d" + } + } + } + } %} + + +{% do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %} +{% do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%} +{% endfor %} +{% endif %} +{% endif %} +{% endfor %} diff --git a/salt/elasticfleet/integration-defaults.map.jinja b/salt/elasticfleet/integration-defaults.map.jinja index f85a95ec9..eeb85123a 100644 --- a/salt/elasticfleet/integration-defaults.map.jinja +++ b/salt/elasticfleet/integration-defaults.map.jinja @@ -59,8 +59,8 @@ {# skip core integrations #} {% elif pkg.name not in CORE_ESFLEET_PACKAGES %} {# generate defaults for each integration #} -{% if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %} -{% for pattern in pkg.es_index_patterns %} +{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %} +{% for pattern in pkg.dataStreams %} {% if "metrics-" in pattern.name %} {% set integration_type = "metrics-" %} {% elif "logs-" in pattern.name %} @@ -75,44 +75,27 @@ {% if component_name in WEIRD_INTEGRATIONS %} {% set component_name = WEIRD_INTEGRATIONS[component_name] %} {% endif %} - -{# create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed - eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #} -{% set custom_component_name = component_name %} - -{# duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #} -{% set generic_integration_type = integration_type %} - {# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #} {% set component_name_x = component_name.replace(".","_x_") %} {# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #} {% set integration_key = "so-" ~ integration_type ~ component_name_x %} -{# if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #} -{% if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %} -{# these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #} -{% set index_pattern = integration_type ~ component_name ~ "-*" %} -{# includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #} -{% set component_name = "filestream.generic" %} -{% set generic_integration_type = "logs-" %} -{% endif %} - {# Default integration settings #} {% set integration_defaults = { "index_sorting": false, "index_template": { - "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], "data_stream": { "allow_custom_routing": false, "hidden": false }, - "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"], + "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], "index_patterns": [index_pattern], "priority": 501, "template": { "settings": { "index": { - "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"}, + "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, "number_of_replicas": 0 } } diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common index 1a597b1db..92532082a 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common @@ -135,9 +135,33 @@ elastic_fleet_bulk_package_install() { fi } -elastic_fleet_installed_packages() { - if ! fleet_api "epm/packages/installed?perPage=500"; then +elastic_fleet_get_package_list_by_type() { + if ! output=$(fleet_api "epm/packages"); then return 1 + else + is_integration=$(jq '[.items[] | select(.type=="integration") | .name ]' <<< "$output") + is_input=$(jq '[.items[] | select(.type=="input") | .name ]' <<< "$output") + is_content=$(jq '[.items[] | select(.type=="content") | .name ]' <<< "$output") + jq -n --argjson is_integration "${is_integration:-[]}" \ + --argjson is_input "${is_input:-[]}" \ + --argjson is_content "${is_content:-[]}" \ + '{"integration": $is_integration,"input": $is_input, "content": $is_content}' + fi +} +elastic_fleet_installed_packages_components() { + package_type=${1,,} + if [[ "$package_type" != "integration" && "$package_type" != "input" && "$package_type" != "content" ]]; then + echo "Error: Invalid package type ${package_type}. Valid types are 'integration', 'input', or 'content'." + return 1 + fi + + packages_by_type=$(elastic_fleet_get_package_list_by_type) + packages=$(jq --arg package_type "$package_type" '.[$package_type]' <<< "$packages_by_type") + + if ! output=$(fleet_api "epm/packages/installed?perPage=500"); then + return 1 + else + jq -c --argjson packages "$packages" '[.items[] | select(.name | IN($packages[])) | {name: .name, dataStreams: .dataStreams}]' <<< "$output" fi } diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade index 0729531d3..aafc9c368 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade @@ -6,6 +6,11 @@ . /usr/sbin/so-common {%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} +{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} +{# Optionally override Elasticsearch version for Elastic Agent patch releases #} +{%- if ELASTICFLEETDEFAULTS.elasticfleet.patch_version is defined %} +{%- do ELASTICSEARCHDEFAULTS.update({'elasticsearch': {'version': ELASTICFLEETDEFAULTS.elasticfleet.patch_version}}) %} +{%- endif %} # Only run on Managers if ! is_manager_node; then diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load index 8c0f627ef..ab38b7065 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load @@ -18,7 +18,9 @@ INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json -PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json +INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json +INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json +CONTENT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_content_package_components.json COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json PENDING_UPDATE=false @@ -179,10 +181,13 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then else echo "Elastic integrations don't appear to need installation/updating..." fi - # Write out file for generating index/component/ilm templates - if latest_installed_package_list=$(elastic_fleet_installed_packages); then - echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS - fi + # Write out file for generating index/component/ilm templates, keeping each package type separate + for package_type in "INTEGRATION" "INPUT" "CONTENT"; do + if latest_installed_package_list=$(elastic_fleet_installed_packages_components "$package_type"); then + outfile="${package_type}_PACKAGE_COMPONENTS" + echo $latest_installed_package_list > "${!outfile}" + fi + done if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then # Refresh installed component template list latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.') diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index 41ef02164..8a4674c71 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -66,6 +66,8 @@ so-elasticsearch-ilm-policy-load-script: - group: 939 - mode: 754 - template: jinja + - defaults: + GLOBALS: {{ GLOBALS }} - show_changes: False so-elasticsearch-pipelines-script: @@ -91,6 +93,13 @@ estemplatedir: - group: 939 - makedirs: True +esaddontemplatedir: + file.directory: + - name: /opt/so/conf/elasticsearch/templates/addon-index + - user: 930 + - group: 939 + - makedirs: True + esrolesdir: file.directory: - name: /opt/so/conf/elasticsearch/roles diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index d0ab0f959..6fb795bce 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,6 +1,6 @@ elasticsearch: enabled: false - version: 9.0.8 + version: 9.3.3 index_clean: true vm: max_map_count: 1048576 diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 29ab80329..f4031ee5d 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -10,8 +10,10 @@ {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} -{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} -{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} +{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %} +{% if GLOBALS.role != 'so-heavynode' %} +{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} +{% endif %} include: - ca @@ -117,40 +119,52 @@ escomponenttemplates: - onchanges_in: - file: so-elasticsearch-templates-reload - show_changes: False - -# Auto-generate templates from defaults file + +# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory +so_index_template_dir: + file.directory: + - name: /opt/so/conf/elasticsearch/templates/index + - clean: True + {%- if SO_MANAGED_INDICES %} + - require: + {%- for index in SO_MANAGED_INDICES %} + - file: so_index_template_{{index}} + {%- endfor %} + {%- endif %} + +# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml) +# These index templates are for the core SO datasets and are always required {% for index, settings in ES_INDEX_SETTINGS.items() %} - {% if settings.index_template is defined %} -es_index_template_{{index}}: +{% if settings.index_template is defined %} +so_index_template_{{index}}: file.managed: - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json - source: salt://elasticsearch/base-template.json.jinja - defaults: - TEMPLATE_CONFIG: {{ settings.index_template }} + TEMPLATE_CONFIG: {{ settings.index_template }} - template: jinja - - show_changes: False - onchanges_in: - file: so-elasticsearch-templates-reload {% endif %} {% endfor %} -{% if TEMPLATES %} -# Sync custom templates to /opt/so/conf/elasticsearch/templates -{% for TEMPLATE in TEMPLATES %} -es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: +{% if GLOBALS.role != "so-heavynode" %} +# Auto-generate optional index templates for integration | input | content packages +# These index templates are not used by default (until user adds package to an agent policy). +# Pre-configured with standard defaults, and incorporated into SOC configuration for user customization. +{% for index,settings in ALL_ADDON_SETTINGS.items() %} +{% if settings.index_template is defined %} +addon_index_template_{{index}}: file.managed: - - source: salt://elasticsearch/templates/index/{{TEMPLATE}} -{% if 'jinja' in TEMPLATE.split('.')[-1] %} - - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}} + - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ settings.index_template }} - template: jinja -{% else %} - - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}} -{% endif %} - - user: 930 - - group: 939 - show_changes: False - onchanges_in: - - file: so-elasticsearch-templates-reload + - file: addon-elasticsearch-templates-reload +{% endif %} {% endfor %} {% endif %} @@ -165,6 +179,7 @@ so-es-cluster-settings: - file: elasticsearch_sbin_jinja {% endif %} +# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml) so-elasticsearch-ilm-policy-load: cmd.run: - name: /usr/sbin/so-elasticsearch-ilm-policy-load @@ -179,9 +194,18 @@ so-elasticsearch-templates-reload: file.absent: - name: /opt/so/state/estemplates.txt +addon-elasticsearch-templates-reload: + file.absent: + - name: /opt/so/state/addon_estemplates.txt + +# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script so-elasticsearch-templates: cmd.run: +{%- if GLOBALS.role == "so-heavynode" %} + - name: /usr/sbin/so-elasticsearch-templates-load --heavynode +{%- else %} - name: /usr/sbin/so-elasticsearch-templates-load +{%- endif %} - cwd: /opt/so - template: jinja - require: diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1 b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 similarity index 72% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1 rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 index d3354f363..1ea828514 100644 --- a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1 +++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 @@ -10,24 +10,28 @@ "processors": [ { "set": { + "tag": "set_ecs_version_f5923549", "field": "ecs.version", "value": "8.17.0" } }, { "set": { + "tag": "set_observer_vendor_ad9d35cc", "field": "observer.vendor", "value": "netgate" } }, { "set": { + "tag": "set_observer_type_5dddf3ba", "field": "observer.type", "value": "firewall" } }, { "rename": { + "tag": "rename_message_to_event_original_56a77271", "field": "message", "target_field": "event.original", "ignore_missing": true, @@ -36,12 +40,14 @@ }, { "set": { + "tag": "set_event_kind_de80643c", "field": "event.kind", "value": "event" } }, { "set": { + "tag": "set_event_timezone_4ca44cac", "field": "event.timezone", "value": "{{{_tmp.tz_offset}}}", "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'" @@ -49,6 +55,7 @@ }, { "grok": { + "tag": "grok_event_original_27d9c8c7", "description": "Parse syslog header", "field": "event.original", "patterns": [ @@ -72,6 +79,7 @@ }, { "date": { + "tag": "date__tmp_timestamp8601_to_timestamp_6ac9d3ce", "if": "ctx._tmp.timestamp8601 != null", "field": "_tmp.timestamp8601", "target_field": "@timestamp", @@ -82,6 +90,7 @@ }, { "date": { + "tag": "date__tmp_timestamp_to_timestamp_f21e536e", "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null", "field": "_tmp.timestamp", "target_field": "@timestamp", @@ -95,6 +104,7 @@ }, { "grok": { + "tag": "grok_process_name_cef3d489", "description": "Set Event Provider", "field": "process.name", "patterns": [ @@ -107,71 +117,83 @@ }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-firewall", + "tag": "pipeline_e16851a7", + "name": "logs-pfsense.log-1.25.2-firewall", "if": "ctx.event.provider == 'filterlog'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-openvpn", + "tag": "pipeline_828590b5", + "name": "logs-pfsense.log-1.25.2-openvpn", "if": "ctx.event.provider == 'openvpn'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-ipsec", + "tag": "pipeline_9d37039c", + "name": "logs-pfsense.log-1.25.2-ipsec", "if": "ctx.event.provider == 'charon'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-dhcp", - "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)" + "tag": "pipeline_ad56bbca", + "name": "logs-pfsense.log-1.25.2-dhcp", + "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-unbound", + "tag": "pipeline_dd85553d", + "name": "logs-pfsense.log-1.25.2-unbound", "if": "ctx.event.provider == 'unbound'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-haproxy", + "tag": "pipeline_720ed255", + "name": "logs-pfsense.log-1.25.2-haproxy", "if": "ctx.event.provider == 'haproxy'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-php-fpm", + "tag": "pipeline_456beba5", + "name": "logs-pfsense.log-1.25.2-php-fpm", "if": "ctx.event.provider == 'php-fpm'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-squid", + "tag": "pipeline_a0d89375", + "name": "logs-pfsense.log-1.25.2-squid", "if": "ctx.event.provider == 'squid'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-snort", + "tag": "pipeline_c2f1ed55", + "name": "logs-pfsense.log-1.25.2-snort", "if": "ctx.event.provider == 'snort'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-suricata", + "tag":"pipeline_33db1c9e", + "name": "logs-pfsense.log-1.25.2-suricata", "if": "ctx.event.provider == 'suricata'" } }, { "drop": { - "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)" + "tag": "drop_9d7c46f8", + "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dnsmasq-dhcp\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)" } }, { "append": { + "tag": "append_event_category_4780a983", "field": "event.category", "value": "network", "if": "ctx.network != null" @@ -179,6 +201,7 @@ }, { "convert": { + "tag": "convert_source_address_to_source_ip_f5632a20", "field": "source.address", "target_field": "source.ip", "type": "ip", @@ -188,6 +211,7 @@ }, { "convert": { + "tag": "convert_destination_address_to_destination_ip_f1388f0c", "field": "destination.address", "target_field": "destination.ip", "type": "ip", @@ -197,6 +221,7 @@ }, { "set": { + "tag": "set_network_type_1f1d940a", "field": "network.type", "value": "ipv6", "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")" @@ -204,6 +229,7 @@ }, { "set": { + "tag": "set_network_type_69deca38", "field": "network.type", "value": "ipv4", "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")" @@ -211,6 +237,7 @@ }, { "geoip": { + "tag": "geoip_source_ip_to_source_geo_da2e41b2", "field": "source.ip", "target_field": "source.geo", "ignore_missing": true @@ -218,6 +245,7 @@ }, { "geoip": { + "tag": "geoip_destination_ip_to_destination_geo_ab5e2968", "field": "destination.ip", "target_field": "destination.geo", "ignore_missing": true @@ -225,6 +253,7 @@ }, { "geoip": { + "tag": "geoip_source_ip_to_source_as_28d69883", "ignore_missing": true, "database_file": "GeoLite2-ASN.mmdb", "field": "source.ip", @@ -237,6 +266,7 @@ }, { "geoip": { + "tag": "geoip_destination_ip_to_destination_as_8a007787", "database_file": "GeoLite2-ASN.mmdb", "field": "destination.ip", "target_field": "destination.as", @@ -249,6 +279,7 @@ }, { "rename": { + "tag": "rename_source_as_asn_to_source_as_number_a917047d", "field": "source.as.asn", "target_field": "source.as.number", "ignore_missing": true @@ -256,6 +287,7 @@ }, { "rename": { + "tag": "rename_source_as_organization_name_to_source_as_organization_name_f1362d0b", "field": "source.as.organization_name", "target_field": "source.as.organization.name", "ignore_missing": true @@ -263,6 +295,7 @@ }, { "rename": { + "tag": "rename_destination_as_asn_to_destination_as_number_3b459fcd", "field": "destination.as.asn", "target_field": "destination.as.number", "ignore_missing": true @@ -270,6 +303,7 @@ }, { "rename": { + "tag": "rename_destination_as_organization_name_to_destination_as_organization_name_814bd459", "field": "destination.as.organization_name", "target_field": "destination.as.organization.name", "ignore_missing": true @@ -277,12 +311,14 @@ }, { "community_id": { + "tag": "community_id_d2308e7a", "target_field": "network.community_id", "ignore_failure": true } }, { "grok": { + "tag": "grok_observer_ingress_interface_name_968018d3", "field": "observer.ingress.interface.name", "patterns": [ "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}" @@ -293,6 +329,7 @@ }, { "set": { + "tag": "set_network_vlan_id_efd4d96a", "field": "network.vlan.id", "copy_from": "observer.ingress.vlan.id", "ignore_empty_value": true @@ -300,6 +337,7 @@ }, { "append": { + "tag": "append_related_ip_c1a6356b", "field": "related.ip", "value": "{{{destination.ip}}}", "allow_duplicates": false, @@ -308,6 +346,7 @@ }, { "append": { + "tag": "append_related_ip_8121c591", "field": "related.ip", "value": "{{{source.ip}}}", "allow_duplicates": false, @@ -316,6 +355,7 @@ }, { "append": { + "tag": "append_related_ip_53b62ed8", "field": "related.ip", "value": "{{{source.nat.ip}}}", "allow_duplicates": false, @@ -324,6 +364,7 @@ }, { "append": { + "tag": "append_related_hosts_6f162628", "field": "related.hosts", "value": "{{{destination.domain}}}", "if": "ctx.destination?.domain != null" @@ -331,6 +372,7 @@ }, { "append": { + "tag": "append_related_user_c036eec2", "field": "related.user", "value": "{{{user.name}}}", "if": "ctx.user?.name != null" @@ -338,6 +380,7 @@ }, { "set": { + "tag": "set_network_direction_cb1e3125", "field": "network.direction", "value": "{{{network.direction}}}bound", "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/" @@ -345,6 +388,7 @@ }, { "remove": { + "tag": "remove_a82e20f2", "field": [ "_tmp" ], @@ -353,11 +397,21 @@ }, { "script": { + "tag": "script_a7f2c062", "lang": "painless", "description": "This script processor iterates over the whole document to remove fields with null values.", "source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n" } }, + { + "append": { + "tag": "append_preserve_original_event_on_error", + "field": "tags", + "value": "preserve_original_event", + "allow_duplicates": false, + "if": "ctx.error?.message != null" + } + }, { "pipeline": { "name": "global@custom", @@ -405,7 +459,14 @@ { "append": { "field": "error.message", - "value": "{{{ _ingest.on_failure_message }}}" + "value": "Processor '{{{ _ingest.on_failure_processor_type }}}' {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'" + } + }, + { + "append": { + "field": "tags", + "value": "preserve_original_event", + "allow_duplicates": false } } ] diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1-suricata b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2-suricata similarity index 100% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1-suricata rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2-suricata diff --git a/salt/elasticsearch/files/log4j2.properties b/salt/elasticsearch/files/log4j2.properties index b29378d6a..050071581 100644 --- a/salt/elasticsearch/files/log4j2.properties +++ b/salt/elasticsearch/files/log4j2.properties @@ -45,3 +45,7 @@ appender.rolling_json.strategy.action.condition.nested_condition.age = 1D rootLogger.level = info rootLogger.appenderRef.rolling.ref = rolling rootLogger.appenderRef.rolling_json.ref = rolling_json + +# Suppress NotEntitledException WARNs (ES 9.3.3 bug) +logger.entitlement_security.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.x-pack-security.org.elasticsearch.security.org.elasticsearch.xpack.security +logger.entitlement_security.level = error \ No newline at end of file diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 2563f8e23..e66057775 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -14,15 +14,42 @@ {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %} +{% set ALL_ADDON_INTEGRATION_DEFAULTS = {} %} +{% set ALL_ADDON_SETTINGS_ORIG = {} %} +{% set ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES = {} %} +{% set ALL_ADDON_SETTINGS = {} %} {# start generation of integration default index_settings #} -{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %} -{% set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %} -{% if check_package_components.size > 1 %} -{% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %} -{% for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %} -{% do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %} -{% endfor %} -{% endif%} +{% if salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %} +{# import integration type defaults #} +{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %} +{% set check_integration_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %} +{% if check_integration_package_components.size > 1 %} +{% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %} +{% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INTEGRATION_DEFAULTS) %} +{% endif %} +{% endif %} + +{# import input type defaults #} +{% if salt['file.file_exists']('/opt/so/state/esfleet_input_package_components.json') %} +{% set check_input_package_components = salt['file.stats']('/opt/so/state/esfleet_input_package_components.json') %} +{% if check_input_package_components.size > 1 %} +{% from 'elasticfleet/input-defaults.map.jinja' import ADDON_INPUT_INTEGRATION_DEFAULTS %} +{% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INPUT_INTEGRATION_DEFAULTS) %} +{% endif %} +{% endif %} + +{# import content type defaults #} +{% if salt['file.file_exists']('/opt/so/state/esfleet_content_package_components.json') %} +{% set check_content_package_components = salt['file.stats']('/opt/so/state/esfleet_content_package_components.json') %} +{% if check_content_package_components.size > 1 %} +{% from 'elasticfleet/content-defaults.map.jinja' import ADDON_CONTENT_INTEGRATION_DEFAULTS %} +{% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_CONTENT_INTEGRATION_DEFAULTS) %} +{% endif %} +{% endif %} + +{% for index, settings in ALL_ADDON_INTEGRATION_DEFAULTS.items() %} +{% do ALL_ADDON_SETTINGS_ORIG.update({index: settings}) %} +{% endfor %} {% endif %} {# end generation of integration default index_settings #} @@ -31,25 +58,33 @@ {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} {% endfor %} +{% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %} +{% for index in ALL_ADDON_SETTINGS_ORIG.keys() %} +{% do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} +{% endfor %} +{% endif %} + {% set ES_INDEX_SETTINGS = {} %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} -{% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %} +{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS) %} + +{% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} +{% for index, settings in GLOBAL_OVERRIDES.items() %} {# prevent this action from being performed on custom defined indices. #} {# the custom defined index is not present in either of the dictionaries and fails to reder. #} -{% if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %} +{% if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %} {# dont merge policy from the global_overrides if policy isn't defined in the original index settingss #} {# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #} -{% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %} +{% if not DEFINED_SETTINGS[index].policy is defined and GLOBAL_OVERRIDES[index].policy is defined %} +{% do GLOBAL_OVERRIDES[index].pop('policy') %} {% endif %} {# this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #} -{% if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} -{% for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %} -{% if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %} +{% if GLOBAL_OVERRIDES[index].policy is defined %} +{% for phase in GLOBAL_OVERRIDES[index].policy.phases.copy() %} +{% if DEFINED_SETTINGS[index].policy.phases[phase] is not defined %} +{% do GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %} {% endif %} {% endfor %} {% endif %} @@ -111,5 +146,14 @@ {% endfor %} {% endif %} -{% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %} +{% do FINAL_INDEX_SETTINGS.update({index | replace("_x_", "."): GLOBAL_OVERRIDES[index]}) %} {% endfor %} +{% endmacro %} + +{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS) }} +{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS) }} + +{% set SO_MANAGED_INDICES = [] %} +{% for index, settings in ES_INDEX_SETTINGS.items() %} +{% do SO_MANAGED_INDICES.append(index) %} +{% endfor %} \ No newline at end of file diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list index 2fccce9cb..6946e30da 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list @@ -6,8 +6,19 @@ # Elastic License 2.0. . /usr/sbin/so-common -if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template | jq '.component_templates[] |.name'| sort + +if [[ -z "$1" ]]; then + if output=$(so-elasticsearch-query "_component_template" --retry 3 --retry-delay 1 --fail); then + jq '[.component_templates[] | .name] | sort' <<< "$output" + else + echo "Failed to retrieve component templates from Elasticsearch." + exit 1 + fi else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template/$1 | jq -fi + if output=$(so-elasticsearch-query "_component_template/$1" --retry 3 --retry-delay 1 --fail); then + jq <<< "$output" + else + echo "Failed to retrieve component template '$1' from Elasticsearch." + exit 1 + fi +fi \ No newline at end of file diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load new file mode 100755 index 000000000..840639a32 --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -0,0 +1,253 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +SO_STATEFILE_SUCCESS=/opt/so/state/estemplates.txt +ADDON_STATEFILE_SUCCESS=/opt/so/state/addon_estemplates.txt +ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates" +SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index" +ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index" +SO_LOAD_FAILURES=0 +ADDON_LOAD_FAILURES=0 +SO_LOAD_FAILURES_NAMES=() +ADDON_LOAD_FAILURES_NAMES=() +IS_HEAVYNODE="false" +FORCE="false" +VERBOSE="false" +SHOULD_EXIT_ON_FAILURE="true" + +# If soup is running, ignore errors +pgrep soup >/dev/null && SHOULD_EXIT_ON_FAILURE="false" + +while [[ $# -gt 0 ]]; do + case "$1" in + --heavynode) + IS_HEAVYNODE="true" + ;; + --force) + FORCE="true" + ;; + --verbose) + VERBOSE="true" + ;; + *) + echo "Usage: $0 [options]" + echo "Options:" + echo " --heavynode Only loads index templates specific to heavynodes" + echo " --force Force reload all templates regardless of statefiles (default: false)" + echo " --verbose Enable verbose output" + exit 1 + ;; + esac + shift +done + +load_template() { + local uri="$1" + local file="$2" + + echo "Loading template file $file" + if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then + echo "$output" + + return 1 + + elif [[ "$VERBOSE" == "true" ]]; then + echo "$output" + fi + +} + +check_required_component_template_exists() { + local required + local missing + local file=$1 + + required=$(jq '[((.composed_of //[]) - (.ignore_missing_component_templates // []))[]]' "$file") + missing=$(jq -n --argjson required "$required" --argjson component_templates "$component_templates" '(($required) - ($component_templates))') + + if [[ $(jq length <<<"$missing") -gt 0 ]]; then + + return 1 + fi +} + +check_heavynode_compatiable_index_template() { + # The only templates that are relevant to heavynodes are from datasets defined in elasticagent/files/elastic-agent.yml.jinja. + # Heavynodes do not have fleet server packages installed and do not support elastic agents reporting directly to them. + local -A heavynode_index_templates=( + ["so-import"]=1 + ["so-syslog"]=1 + ["so-logs-soc"]=1 + ["so-suricata"]=1 + ["so-suricata.alerts"]=1 + ["so-zeek"]=1 + ["so-strelka"]=1 + ) + + local template_name="$1" + + if [[ ! -v heavynode_index_templates["$template_name"] ]]; then + + return 1 + fi + +} + +load_component_templates() { + local printed_name="$1" + local pattern="${ELASTICSEARCH_TEMPLATES_DIR}/component/$2" + local append_mappings="${3:-"false"}" + + # current state of nullglob shell option + shopt -q nullglob && nullglob_set=1 || nullglob_set=0 + + shopt -s nullglob + echo -e "\nLoading $printed_name component templates...\n" + for component in "$pattern"/*.json; do + tmpl_name=$(basename "${component%.json}") + + if [[ "$append_mappings" == "true" ]]; then + # avoid duplicating "-mappings" if it already exists in the component template filename + tmpl_name="${tmpl_name%-mappings}-mappings" + fi + + if ! load_template "_component_template/${tmpl_name}" "$component"; then + SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1)) + SO_LOAD_FAILURES_NAMES+=("$component") + fi + done + + # restore nullglob shell option if needed + if [[ $nullglob_set -eq 1 ]]; then + shopt -u nullglob + fi +} + +check_elasticsearch_responsive() { + # Cannot load templates if Elasticsearch is not responding. + # NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the + # script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state. + retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" || + fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot." +} + +if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]]; then + check_elasticsearch_responsive + + if [[ "$IS_HEAVYNODE" == "false" ]]; then + # TODO: Better way to check if fleet server is installed vs checking for Elastic Defend component template. + fleet_check="logs-endpoint.alerts@package" + if ! so-elasticsearch-query "_component_template/$fleet_check" --output /dev/null --retry 5 --retry-delay 3 --fail; then + # This check prevents so-elasticsearch-templates-load from running before so-elastic-fleet-setup has run. + echo -e "\nPackage $fleet_check not yet installed. Fleet Server may not be fully configured yet." + # Fleet Server is required because some SO index templates depend on components installed via + # specific integrations eg Elastic Defend. These are components that we do not manually create / manage + # via /opt/so/saltstack/salt/elasticsearch/templates/component/ + + exit 0 + fi + fi + + # load_component_templates "Name" "directory" "append '-mappings'?" + load_component_templates "ECS" "ecs" "true" + load_component_templates "Elastic Agent" "elastic-agent" + load_component_templates "Security Onion" "so" + + component_templates=$(so-elasticsearch-component-templates-list) + echo -e "Loading Security Onion index templates...\n" + for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do + tmpl_name=$(basename "${so_idx_tmpl%-template.json}") + + if [[ "$IS_HEAVYNODE" == "true" ]]; then + # TODO: Better way to load only heavynode specific templates + if ! check_heavynode_compatiable_index_template "$tmpl_name"; then + if [[ "$VERBOSE" == "true" ]]; then + echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template." + fi + + continue + fi + fi + + if check_required_component_template_exists "$so_idx_tmpl"; then + if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then + SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1)) + SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl") + fi + else + echo "Skipping over $so_idx_tmpl due to missing required component template(s)." + SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1)) + SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl") + + continue + fi + done + + if [[ $SO_LOAD_FAILURES -eq 0 ]]; then + echo "All Security Onion core templates loaded successfully." + + touch "$SO_STATEFILE_SUCCESS" + else + echo "Encountered $SO_LOAD_FAILURES failure(s) loading templates:" + for failed_template in "${SO_LOAD_FAILURES_NAMES[@]}"; do + echo " - $failed_template" + done + if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then + fail "Failed to load all Security Onion core templates successfully." + fi + fi +else + + echo "Security Onion core templates already loaded" +fi + +# Start loading addon templates +if [[ (-d "$ADDON_TEMPLATES_DIR" && -f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && ! -f "$ADDON_STATEFILE_SUCCESS") || (-d "$ADDON_TEMPLATES_DIR" && "$IS_HEAVYNODE" == "false" && "$FORCE" == "true") ]]; then + + check_elasticsearch_responsive + + echo -e "\nLoading addon integration index templates...\n" + component_templates=$(so-elasticsearch-component-templates-list) + + for addon_idx_tmpl in "${ADDON_TEMPLATES_DIR}"/*.json; do + tmpl_name=$(basename "${addon_idx_tmpl%-template.json}") + + if check_required_component_template_exists "$addon_idx_tmpl"; then + if ! load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl"; then + ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1)) + ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl") + fi + else + echo "Skipping over $addon_idx_tmpl due to missing required component template(s)." + ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1)) + ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl") + + continue + fi + done + + if [[ $ADDON_LOAD_FAILURES -eq 0 ]]; then + echo "All addon integration templates loaded successfully." + + touch "$ADDON_STATEFILE_SUCCESS" + else + echo "Encountered $ADDON_LOAD_FAILURES failure(s) loading addon integration templates:" + for failed_template in "${ADDON_LOAD_FAILURES_NAMES[@]}"; do + echo " - $failed_template" + done + if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then + fail "Failed to load all addon integration templates successfully." + fi + fi + +elif [[ ! -f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" ]]; then + echo "Skipping loading addon integration templates until Security Onion core templates have been loaded." + +elif [[ -f "$ADDON_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && "$FORCE" == "false" ]]; then + echo "Addon integration templates already loaded" +fi diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index 04a7a8ab0..7988c1905 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -7,6 +7,9 @@ . /usr/sbin/so-common {%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} +{%- if GLOBALS.role != "so-heavynode" %} +{%- from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} +{%- endif %} {%- for index, settings in ES_INDEX_SETTINGS.items() %} {%- if settings.policy is defined %} @@ -33,3 +36,13 @@ {%- endif %} {%- endfor %} echo +{%- if GLOBALS.role != "so-heavynode" %} +{%- for index, settings in ALL_ADDON_SETTINGS.items() %} +{%- if settings.policy is defined %} + echo + echo "Setting up {{ index }}-logs policy..." + curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' + echo +{%- endif %} +{%- endfor %} +{%- endif %} diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load deleted file mode 100755 index ad3fe1344..000000000 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ /dev/null @@ -1,165 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. -{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} -{% from 'vars/globals.map.jinja' import GLOBALS %} - -STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt -STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt - -if [[ -f $STATE_FILE_INITIAL ]]; then - # The initial template load has already run. As this is a subsequent load, all dependencies should - # already be satisified. Therefore, immediately exit/abort this script upon any template load failure - # since this is an unrecoverable failure. - should_exit_on_failure=1 -else - # This is the initial template load, and there likely are some components not yet setup in Elasticsearch. - # Therefore load as many templates as possible at this time and if an error occurs proceed to the next - # template. But if at least one template fails to load do not mark the templates as having been loaded. - # This will allow the next load to resume the load of the templates that failed to load initially. - should_exit_on_failure=0 - echo "This is the initial template load" -fi - -# If soup is running, ignore errors -pgrep soup > /dev/null && should_exit_on_failure=0 - -load_failures=0 - -load_template() { - uri=$1 - file=$2 - - echo "Loading template file $i" - if ! retry 3 1 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then - if [[ $should_exit_on_failure -eq 1 ]]; then - fail "Could not load template file: $file" - else - load_failures=$((load_failures+1)) - echo "Incremented load failure counter: $load_failures" - fi - fi -} - -if [ ! -f $STATE_FILE_SUCCESS ]; then - echo "State file $STATE_FILE_SUCCESS not found. Running so-elasticsearch-templates-load." - - . /usr/sbin/so-common - - {% if GLOBALS.role != 'so-heavynode' %} - if [ -f /usr/sbin/so-elastic-fleet-common ]; then - . /usr/sbin/so-elastic-fleet-common - fi - {% endif %} - - default_conf_dir=/opt/so/conf - - # Define a default directory to load pipelines from - ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/" - - {% if GLOBALS.role == 'so-heavynode' %} - file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json" - {% else %} - file="/usr/sbin/so-elastic-fleet-common" - {% endif %} - - if [ -f "$file" ]; then - # Wait for ElasticSearch to initialize - echo -n "Waiting for ElasticSearch..." - retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" - {% if GLOBALS.role != 'so-heavynode' %} - TEMPLATE="logs-endpoint.alerts@package" - INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name) - if [ "$INSTALLED" != "$TEMPLATE" ]; then - echo - echo "Packages not yet installed." - echo - exit 0 - fi - {% endif %} - - touch $STATE_FILE_INITIAL - - cd ${ELASTICSEARCH_TEMPLATES}/component/ecs - - echo "Loading ECS component templates..." - for i in *; do - TEMPLATE=$(echo $i | cut -d '.' -f1) - load_template "_component_template/${TEMPLATE}-mappings" "$i" - done - echo - - cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent - - echo "Loading Elastic Agent component templates..." - {% if GLOBALS.role == 'so-heavynode' %} - component_pattern="so-*" - {% else %} - component_pattern="*" - {% endif %} - for i in $component_pattern; do - TEMPLATE=${i::-5} - load_template "_component_template/$TEMPLATE" "$i" - done - echo - - # Load SO-specific component templates - cd ${ELASTICSEARCH_TEMPLATES}/component/so - - echo "Loading Security Onion component templates..." - for i in *; do - TEMPLATE=$(echo $i | cut -d '.' -f1); - load_template "_component_template/$TEMPLATE" "$i" - done - echo - - # Load SO index templates - cd ${ELASTICSEARCH_TEMPLATES}/index - - echo "Loading Security Onion index templates..." - shopt -s extglob - {% if GLOBALS.role == 'so-heavynode' %} - pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*|*endpoint*|*elasticsearch*|*generic*|*fleet_server*|*soc*)" - {% else %} - pattern="*" - {% endif %} - # Index templates will be skipped if the following conditions are met: - # 1. The template is part of the "so-logs-" template group - # 2. The template name does not correlate to at least one existing component template - # In this situation, the script will treat the skipped template as a temporary failure - # and allow the templates to be loaded again on the next run or highstate, whichever - # comes first. - COMPONENT_LIST=$(so-elasticsearch-component-templates-list) - for i in $pattern; do - TEMPLATE=${i::-14} - COMPONENT_PATTERN=${TEMPLATE:3} - MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery") - if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ \.generic|logs-winlog\.winlog ]]; then - load_failures=$((load_failures+1)) - echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures" - else - load_template "_index_template/$TEMPLATE" "$i" - fi - done - else - {% if GLOBALS.role == 'so-heavynode' %} - echo "Common template does not exist. Exiting..." - {% else %} - echo "Elastic Fleet not configured. Exiting..." - {% endif %} - exit 0 - fi - - cd - >/dev/null - - if [[ $load_failures -eq 0 ]]; then - echo "All templates loaded successfully" - touch $STATE_FILE_SUCCESS - else - echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate" - fi -else - echo "Templates already loaded" -fi diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index a01d33cb8..3430ef777 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -11,18 +11,14 @@ global: regexFailureMessage: You must enter a valid IP address or CIDR. mdengine: description: Which engine to use for meta data generation. Options are ZEEK and SURICATA. - regex: ^(ZEEK|SURICATA)$ options: - ZEEK - SURICATA - regexFailureMessage: You must enter either ZEEK or SURICATA. global: True pcapengine: description: Which engine to use for generating pcap. Currently only SURICATA is supported. - regex: ^(SURICATA)$ options: - SURICATA - regexFailureMessage: You must enter either SURICATA. global: True ids: description: Which IDS engine to use. Currently only Suricata is supported. @@ -42,11 +38,9 @@ global: advanced: True pipeline: description: Sets which pipeline technology for events to use. The use of Kafka requires a Security Onion Pro license. - regex: ^(REDIS|KAFKA)$ options: - REDIS - KAFKA - regexFailureMessage: You must enter either REDIS or KAFKA. global: True advanced: True repo_host: diff --git a/salt/influxdb/soc_influxdb.yaml b/salt/influxdb/soc_influxdb.yaml index 3dbf0875b..2b6bffe49 100644 --- a/salt/influxdb/soc_influxdb.yaml +++ b/salt/influxdb/soc_influxdb.yaml @@ -85,7 +85,10 @@ influxdb: description: The log level to use for outputting log statements. Allowed values are debug, info, or error. global: True advanced: false - regex: ^(info|debug|error)$ + options: + - info + - debug + - error helpLink: influxdb metrics-disabled: description: If true, the HTTP endpoint that exposes internal InfluxDB metrics will be inaccessible. @@ -140,7 +143,9 @@ influxdb: description: Determines the type of storage used for secrets. Allowed values are bolt or vault. global: True advanced: True - regex: ^(bolt|vault)$ + options: + - bolt + - vault helpLink: influxdb session-length: description: Number of minutes that a user login session can remain authenticated. @@ -260,7 +265,9 @@ influxdb: description: The type of data store to use for HTTP resources. Allowed values are disk or memory. Memory should not be used for production Security Onion installations. global: True advanced: True - regex: ^(disk|memory)$ + options: + - disk + - memory helpLink: influxdb tls-cert: description: The container path to the certificate to use for TLS encryption of the HTTP requests and responses. diff --git a/salt/kafka/soc_kafka.yaml b/salt/kafka/soc_kafka.yaml index b8d0c7c32..85469b8a4 100644 --- a/salt/kafka/soc_kafka.yaml +++ b/salt/kafka/soc_kafka.yaml @@ -128,10 +128,13 @@ kafka: title: ssl.keystore.password sensitive: True helpLink: kafka - ssl_x_keystore_x_type: + ssl_x_keystore_x_type: description: The key store file format. title: ssl.keystore.type - regex: ^(JKS|PKCS12|PEM)$ + options: + - JKS + - PKCS12 + - PEM helpLink: kafka ssl_x_truststore_x_location: description: The trust store file location within the Docker container. @@ -160,7 +163,11 @@ kafka: security_x_protocol: description: 'Broker communication protocol. Options are: SASL_SSL, PLAINTEXT, SSL, SASL_PLAINTEXT' title: security.protocol - regex: ^(SASL_SSL|PLAINTEXT|SSL|SASL_PLAINTEXT) + options: + - SASL_SSL + - PLAINTEXT + - SSL + - SASL_PLAINTEXT helpLink: kafka ssl_x_keystore_x_location: description: The key store file location within the Docker container. @@ -174,7 +181,10 @@ kafka: ssl_x_keystore_x_type: description: The key store file format. title: ssl.keystore.type - regex: ^(JKS|PKCS12|PEM)$ + options: + - JKS + - PKCS12 + - PEM helpLink: kafka ssl_x_truststore_x_location: description: The trust store file location within the Docker container. diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults b/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults index fcb80e606..d0447f514 100755 --- a/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults +++ b/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults @@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http: # Disable certain Features from showing up in the Kibana UI echo echo "Setting up default Kibana Space:" -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","entityManager"]} ' >> /opt/so/log/kibana/misc.log +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log echo diff --git a/salt/kratos/soc_kratos.yaml b/salt/kratos/soc_kratos.yaml index 1cd2728c8..07359bcab 100644 --- a/salt/kratos/soc_kratos.yaml +++ b/salt/kratos/soc_kratos.yaml @@ -21,8 +21,12 @@ kratos: description: "Specify the provider type. Required. Valid values are: auth0, generic, github, google, microsoft" global: True forcedType: string - regex: "auth0|generic|github|google|microsoft" - regexFailureMessage: "Valid values are: auth0, generic, github, google, microsoft" + options: + - auth0 + - generic + - github + - google + - microsoft helpLink: oidc client_id: description: Specify the client ID, also referenced as the application ID. Required. @@ -43,8 +47,9 @@ kratos: description: The source of the subject identifier. Typically 'userinfo'. Only used when provider is 'microsoft'. global: True forcedType: string - regex: me|userinfo - regexFailureMessage: "Valid values are: me, userinfo" + options: + - me + - userinfo helpLink: oidc auth_url: description: Provider's auth URL. Required when provider is 'generic'. diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 2d5ef448e..76b067817 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -132,8 +132,8 @@ function getinstallinfo() { log "ERROR" "Failed to get install info from $MINION_ID" return 1 fi - - export $(echo "$INSTALLVARS" | xargs) + + while read -r var; do export "$var"; done <<< "$INSTALLVARS" if [ $? -ne 0 ]; then log "ERROR" "Failed to source install variables" return 1 diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 0bde8f20e..cc80758fc 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2687,4 +2687,5 @@ soc: lowBalanceColorAlert: 500000 enabled: true adapter: SOAI + charsPerTokenEstimate: 4 diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index c5f96894d..d4e908637 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -761,7 +761,7 @@ soc: required: True - field: origin label: Country of Origin for the Model Training - required: false + required: False - field: contextLimitSmall label: Context Limit (Small) forcedType: int @@ -779,6 +779,10 @@ soc: - field: enabled label: Enabled forcedType: bool + - field: charsPerTokenEstimate + label: Characters per Token Estimate + forcedType: float + required: False apiTimeoutMs: description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI. global: True diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 944e0e34d..f7059b293 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -33,7 +33,7 @@ {% do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %} {# multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #} -{% set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round | int %} +{% set maxfiles = ([1, (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round(0, 'ceil') | int] | max) %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %} {% endif %} diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index c85b876a9..ce6b7d008 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -64,8 +64,10 @@ suricata: helpLink: suricata conditional: description: Set to "all" to record PCAP for all flows. Set to "alerts" to only record PCAP for Suricata alerts. Set to "tag" to only record PCAP for tagged rules. - regex: ^(all|alerts|tag)$ - regexFailureMessage: You must enter either all, alert or tag. + options: + - all + - alerts + - tag helpLink: suricata dir: description: Parent directory to store PCAP. @@ -83,7 +85,9 @@ suricata: advanced: True cluster-type: advanced: True - regex: ^(cluster_flow|cluster_qm)$ + options: + - cluster_flow + - cluster_qm defrag: description: Enable defragmentation of IP packets before processing. forcedType: bool diff --git a/setup/so-setup b/setup/so-setup index 823a379df..46b11fc11 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -219,6 +219,7 @@ if [ -n "$test_profile" ]; then WEBUSER=onionuser@somewhere.invalid WEBPASSWD1=0n10nus3r WEBPASSWD2=0n10nus3r + NODE_DESCRIPTION="${HOSTNAME} - ${install_type} - ${MAINIP}" update_sudoers_for_testing fi