diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index bb79891b6..46717f3e1 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -20,7 +20,7 @@ ], "data_stream.dataset": "import", "custom": "", - "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-1.66.1\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-2.4.1\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-1.66.1\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-1.66.1\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-2.4.1\n- add_fields:\n target: data_stream\n fields:\n dataset: import", + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-1.67.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-2.5.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-1.67.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-1.67.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-2.5.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", "tags": [ "import" ] diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index c3957361a..02e2f7ccb 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,6 +1,6 @@ elasticsearch: enabled: false - version: 8.17.2 + version: 8.17.3 index_clean: true config: action: @@ -2677,6 +2677,59 @@ elasticsearch: settings: index: number_of_replicas: 0 + so-logs-osquery-manager_x_action_x_responses: + index_sorting: false + index_template: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + data_stream: + allow_custom_routing: false + hidden: false + composed_of: + - logs-osquery_manager.action.responses@package + - logs-osquery_manager.action.responses@custom + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + ignore_missing_component_templates: + - logs-osquery_manager.action.responses@custom + index_patterns: + - logs-osquery_manager.action.responses* + priority: 501 + template: + settings: + lifecycle: + name: + so-logs-osquery-manager.action.responses-logs + index: + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-osquery-manager-actions: index_sorting: false index_template: @@ -2695,6 +2748,58 @@ elasticsearch: settings: index: number_of_replicas: 0 + so-logs-osquery-manager_x_result: + index_sorting: false + index_template: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + data_stream: + allow_custom_routing: false + hidden: false + composed_of: + - logs-osquery_manager.result@package + - logs-osquery_manager.result@custom + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + ignore_missing_component_templates: + - logs-osquery_manager.result@custom + index_patterns: + - logs-osquery_manager.result* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-osquery-manager.result-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-soc: close: 30 delete: 365 diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 47013e48f..8420611f2 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -133,7 +133,7 @@ elasticsearch: helpLink: elasticsearch.html cold: min_age: - description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier. regex: ^[0-9]{1,5}d$ forcedType: string global: True @@ -146,10 +146,11 @@ elasticsearch: helpLink: elasticsearch.html warm: min_age: - description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. + description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier. regex: ^[0-9]{1,5}d$ forcedType: string global: True + helpLink: elasticsearch.html actions: set_priority: priority: @@ -159,7 +160,7 @@ elasticsearch: helpLink: elasticsearch.html delete: min_age: - description: Minimum age of index. ex. 90d - This determines when the index should be deleted. + description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion. regex: ^[0-9]{1,5}d$ forcedType: string global: True @@ -288,7 +289,7 @@ elasticsearch: helpLink: elasticsearch.html warm: min_age: - description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. + description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier. regex: ^[0-9]{1,5}d$ forcedType: string global: True @@ -315,7 +316,7 @@ elasticsearch: helpLink: elasticsearch.html cold: min_age: - description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier. regex: ^[0-9]{1,5}d$ forcedType: string global: True @@ -331,7 +332,7 @@ elasticsearch: helpLink: elasticsearch.html delete: min_age: - description: Minimum age of index. This determines when the index should be deleted. + description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion. regex: ^[0-9]{1,5}d$ forcedType: string global: True @@ -370,6 +371,8 @@ elasticsearch: so-logs-httpjson_x_generic: *indexSettings so-logs-osquery-manager-actions: *indexSettings so-logs-osquery-manager-action_x_responses: *indexSettings + so-logs-osquery-manager_x_action_x_responses: *indexSettings + so-logs-osquery-manager_x_result: *indexSettings so-logs-elastic_agent_x_apm_server: *indexSettings so-logs-elastic_agent_x_auditbeat: *indexSettings so-logs-elastic_agent_x_cloudbeat: *indexSettings diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs@custom.json new file mode 100644 index 000000000..61a69003f --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/logs@custom.json @@ -0,0 +1,9 @@ +{ + "template": { + "settings": { + "index": { + "number_of_replicas": "0" + } + } + } +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/elastic-agent/metrics@custom.json b/salt/elasticsearch/templates/component/elastic-agent/metrics@custom.json index 5b459147b..61a69003f 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/metrics@custom.json +++ b/salt/elasticsearch/templates/component/elastic-agent/metrics@custom.json @@ -1,7 +1,9 @@ { "template": { "settings": { - "number_of_replicas": 0 + "index": { + "number_of_replicas": "0" + } } } } \ No newline at end of file diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index 5a349a3c3..15cae92b3 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -12,11 +12,18 @@ global: mdengine: description: Which engine to use for meta data generation. Options are ZEEK and SURICATA. regex: ^(ZEEK|SURICATA)$ + options: + - ZEEK + - SURICATA regexFailureMessage: You must enter either ZEEK or SURICATA. global: True pcapengine: description: Which engine to use for generating pcap. Options are STENO, SURICATA or TRANSITION. regex: ^(STENO|SURICATA|TRANSITION)$ + options: + - STENO + - SURICATA + - TRANSITION regexFailureMessage: You must enter either STENO, SURICATA or TRANSITION. global: True ids: @@ -38,6 +45,9 @@ global: pipeline: description: Sets which pipeline technology for events to use. Currently only Redis is fully supported. Kafka is experimental and requires a Security Onion Pro license. regex: ^(REDIS|KAFKA)$ + options: + - REDIS + - KAFKA regexFailureMessage: You must enter either REDIS or KAFKA. global: True advanced: True diff --git a/salt/kibana/defaults.yaml b/salt/kibana/defaults.yaml index a4be3787f..2446821f1 100644 --- a/salt/kibana/defaults.yaml +++ b/salt/kibana/defaults.yaml @@ -22,7 +22,7 @@ kibana: - default - file migrations: - discardCorruptObjects: "8.17.2" + discardCorruptObjects: "8.17.3" telemetry: enabled: False security: diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 962d1096b..5a8ec840c 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1900,7 +1900,7 @@ soc: query: 'event.module:endpoint | groupby event.dataset | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.name | groupby process.name' - name: Elastic Agent API Events description: API (Application Programming Interface) events from Elastic Agents - query: 'event.dataset:endpoint.events.api | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.name | groupby process.name | groupby process.Ext.api.name' + query: 'event.dataset:endpoint.events.api | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.name | groupby process.name | groupby -sankey process.name process.Ext.api.name | groupby process.Ext.api.name' - name: Elastic Agent File Events description: File events from Elastic Agents query: 'event.dataset:endpoint.events.file | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.name | groupby process.name | groupby event.action | groupby file.path' @@ -2337,6 +2337,7 @@ soc: eventFetchLimit: 500 eventItemsPerPage: 50 groupFetchLimit: 50 + groupItemsPerPage: 10 mostRecentlyUsedLimit: 5 safeStringMaxLength: 100 queryBaseFilter: '_index:"*:so-detection" AND so_kind:detection' diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index d7fcd9644..7153c7b5c 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -60,9 +60,34 @@ soc: - warn - error actions: - description: A list of actions a user can take from the SOC UI against a hunt, alert, and other records. The action must be defined in JSON object format, and contain a "name" key and "links" key. The links is a list of URLs, where the most suitable URL in the list will be the selected URL when the user clicks the action. + description: A list of actions a user can take from the SOC UI against a hunt, alert, and other records. The links is a list of URLs, where the most suitable URL in the list will be the selected URL when the user clicks the action. global: True forcedType: "[]{}" + syntax: json + uiElements: + - field: name + label: Name + required: True + - field: description + label: Description + - field: icon + label: Icon (Example - fa-shuttle-space) + - field: links + label: Links + required: True + forcedType: "[]string" + multiline: True + - field: target + label: Target + - field: jscall + label: JavaScript Call + - field: category + label: Category + options: + - hunt + - alerts + - dashboards + forcedType: "[]string" eventFields: default: &eventFields description: Event fields mappings are defined by the format ":event.module:event.dataset". For example, to customize which fields show for 'syslog' events originating from 'zeek', find the eventField item in the left panel that looks like ':zeek:syslog'. The 'default' entry is used for all events that do not match an existing mapping defined in the list to the left. @@ -492,9 +517,22 @@ soc: description: Number of items to show in the most recently used queries list. Larger values cause default queries to be located further down the list. global: True queries: - description: List of default queries to show in the query list. Each query is represented in JSON object notation, and must include the "name" key and "query" key. + description: List of default queries to show in the query list. global: True forcedType: "[]{}" + syntax: json + uiElements: + - field: name + label: Name + required: True + - field: description + label: Description + - field: query + label: Query + required: True + - field: showSubtitle + label: Show Query in Dropdown. + forcedType: bool queryToggleFilters: description: Customize togglable query filters that apply to all queries. Exclusive toggles will invert the filter if toggled off rather than omitting the filter from the query. global: True