From 9095bc220561c297e86ae0fd8c006cbc023fb1b5 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 19 Sep 2022 15:41:54 +0000 Subject: [PATCH 01/21] Re-establish Elasticsearch cluster (search) settings --- salt/elasticsearch/init.sls | 9 +++++ .../sbin/so-elasticsearch-cluster-settings | 36 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100755 salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-settings diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index bc33598f3..be28a3712 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -346,6 +346,15 @@ append_so-elasticsearch_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-elasticsearch +so-es-cluster-settings: + cmd.run: + - name: /usr/sbin/so-elasticsearch-cluster-settings + - cwd: /opt/so + - template: jinja + - require: + - docker_container: so-elasticsearch + - file: es_sync_scripts + so-elasticsearch-templates: cmd.run: - name: /usr/sbin/so-elasticsearch-templates-load diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-settings b/salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-settings new file mode 100755 index 000000000..a72ccaf56 --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-settings @@ -0,0 +1,36 @@ +#!/bin/bash +{% set ES = salt['pillar.get']('manager:mainip', '') %} +{% set MANAGER = salt['grains.get']('master') %} + +ELASTICSEARCH_PORT=9200 + +# Wait for ElasticSearch to come up, so that we can query for version infromation +echo -n "Waiting for ElasticSearch..." +COUNT=0 +ELASTICSEARCH_CONNECTED="no" +while [[ "$COUNT" -le 30 ]]; do + curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://localhost:"$ELASTICSEARCH_PORT" + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi +done +if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'docker ps' \n -running 'sudo so-elastic-restart'" + echo + + exit +fi + +# Check to see if config already exists +CLUSTER_SETTINGS=$(so-elasticsearch-query _cluster/settings | jq .persistent.cluster.remote) +if [[ ! -z "$CLUSTER_SETTINGS" ]]; then + echo "Applying cross cluster search config..." + so-elasticsearch-query _cluster/settings -d "{\"persistent\": {\"cluster\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" -XPUT +fi From d28a9ecec240dfd5cd2434b2e3755bcdd172ec11 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 19 Sep 2022 13:32:04 -0400 Subject: [PATCH 02/21] Set Dashboard UUID --- salt/grafana/dashboards/common_template.json.jinja | 2 ++ salt/grafana/init.sls | 1 + 2 files changed, 3 insertions(+) diff --git a/salt/grafana/dashboards/common_template.json.jinja b/salt/grafana/dashboards/common_template.json.jinja index 23060a2fb..4c6745c3c 100644 --- a/salt/grafana/dashboards/common_template.json.jinja +++ b/salt/grafana/dashboards/common_template.json.jinja @@ -57,6 +57,8 @@ "title": "{{ TITLE }}", {% if TITLE | lower == 'security onion grid overview' %} "uid": "so_overview", + {% else %} + "uid": "{{ UID }}", {% endif %} "version": 1 } diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls index f71bc3acb..584219906 100644 --- a/salt/grafana/init.sls +++ b/salt/grafana/init.sls @@ -117,6 +117,7 @@ so-grafana-dashboard-folder-delete: TEMPLATES: {{GRAFANA_SETTINGS.dashboards[dashboard].templating.list}} TITLE: {{ GRAFANA_SETTINGS.dashboards[dashboard].get('title', dashboard| capitalize) }} ID: {{ loop.index }} + UID: {{ dashboard }} {% endfor %} so-grafana: From fdffac83e145b968f55e6592419c6fa72fc265f1 Mon Sep 17 00:00:00 2001 From: doug Date: Mon, 19 Sep 2022 14:47:45 -0400 Subject: [PATCH 03/21] sysmon fix by bryant --- salt/elasticsearch/files/ingest/sysmon | 119 ++++++++++-------- .../component/so/so-scan-mappings.json | 32 ++++- salt/soc/defaults.yaml | 86 +++++++++++++ 3 files changed, 181 insertions(+), 56 deletions(-) diff --git a/salt/elasticsearch/files/ingest/sysmon b/salt/elasticsearch/files/ingest/sysmon index e4db4bcb4..5fa0e1005 100644 --- a/salt/elasticsearch/files/ingest/sysmon +++ b/salt/elasticsearch/files/ingest/sysmon @@ -9,61 +9,70 @@ { "set": { "if": "ctx.event?.code == '5'", "field": "event.category", "value": "host,process", "override": true } }, { "set": { "if": "ctx.event?.code == '6'", "field": "event.category", "value": "host,driver", "override": true } }, { "set": { "if": "ctx.event?.code == '22'", "field": "event.category", "value": "network", "override": true } }, - { "set": { "if": "ctx.event?.code == '1'", "field": "event.dataset", "value": "process_creation", "override": true } }, - { "set": { "if": "ctx.event?.code == '2'", "field": "event.dataset", "value": "process_changed_file", "override": true } }, - { "set": { "if": "ctx.event?.code == '3'", "field": "event.dataset", "value": "network_connection", "override": true } }, - { "set": { "if": "ctx.event?.code == '5'", "field": "event.dataset", "value": "process_terminated", "override": true } }, - { "set": { "if": "ctx.event?.code == '6'", "field": "event.dataset", "value": "driver_loaded", "override": true } }, - { "set": { "if": "ctx.event?.code == '7'", "field": "event.dataset", "value": "image_loaded", "override": true } }, - { "set": { "if": "ctx.event?.code == '8'", "field": "event.dataset", "value": "create_remote_thread", "override": true } }, - { "set": { "if": "ctx.event?.code == '9'", "field": "event.dataset", "value": "raw_file_access_read", "override": true } }, - { "set": { "if": "ctx.event?.code == '10'", "field": "event.dataset", "value": "process_access", "override": true } }, - { "set": { "if": "ctx.event?.code == '11'", "field": "event.dataset", "value": "file_create", "override": true } }, - { "set": { "if": "ctx.event?.code == '12'", "field": "event.dataset", "value": "registry_create_delete", "override": true } }, - { "set": { "if": "ctx.event?.code == '13'", "field": "event.dataset", "value": "registry_value_set", "override": true } }, - { "set": { "if": "ctx.event?.code == '14'", "field": "event.dataset", "value": "registry_key_value_rename", "override": true } }, - { "set": { "if": "ctx.event?.code == '15'", "field": "event.dataset", "value": "file_create_stream_hash", "override": true } }, - { "set": { "if": "ctx.event?.code == '16'", "field": "event.dataset", "value": "config_change", "override": true } }, - { "set": { "if": "ctx.event?.code == '22'", "field": "event.dataset", "value": "dns_query", "override": true } }, - { "rename": { "field": "winlog.event_data.SubjectUserName", "target_field": "user.name", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.DestinationHostname", "target_field": "destination.hostname", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.DestinationIp", "target_field": "destination.ip", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.DestinationPort", "target_field": "destination.port", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.image", "target_field": "process.executable", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.Image", "target_field": "process.executable", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.processID", "target_field": "process.pid", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.ProcessId", "target_field": "process.pid", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.processGuid", "target_field": "process.entity_id", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.ProcessGuid", "target_field": "process.entity_id", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.commandLine", "target_field": "process.command_line", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.CommandLine", "target_field": "process.command_line", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.currentDirectory", "target_field": "process.working_directory", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.CurrentDirectory", "target_field": "process.working_directory", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.description", "target_field": "process.pe.description", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.Description", "target_field": "process.pe.description", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.product", "target_field": "process.pe.product", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.Product", "target_field": "process.pe.product", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.company", "target_field": "process.pe.company", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.Company", "target_field": "process.pe.company", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.originalFileName", "target_field": "process.pe.original_file_name", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.OriginalFileName", "target_field": "process.pe.original_file_name", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.fileVersion", "target_field": "process.pe.file_version", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.FileVersion", "target_field": "process.pe.file_version", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.parentCommandLine", "target_field": "process.parent.command_line", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.ParentCommandLine", "target_field": "process.parent.command_line", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.parentImage", "target_field": "process.parent.executable", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.ParentImage", "target_field": "process.parent.executable", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.parentProcessGuid", "target_field": "process.parent.entity_id", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.ParentProcessGuid", "target_field": "process.parent.entity_id", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.parentProcessId", "target_field": "process.ppid", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.ParentProcessId", "target_field": "process.ppid", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.Protocol", "target_field": "network.transport", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.User", "target_field": "user.name", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.SourceHostname", "target_field": "source.hostname", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.SourceIp", "target_field": "source.ip", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.SourcePort", "target_field": "source.port", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.targetFilename", "target_field": "file.target", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.TargetFilename", "target_field": "file.target", "ignore_missing": true } }, + { "set": { "if": "ctx.event?.code == '1'", "field": "event.dataset", "value": "process_creation", "override": true } }, + { "set": { "if": "ctx.event?.code == '2'", "field": "event.dataset", "value": "process_changed_file", "override": true } }, + { "set": { "if": "ctx.event?.code == '3'", "field": "event.dataset", "value": "network_connection", "override": true } }, + { "set": { "if": "ctx.event?.code == '5'", "field": "event.dataset", "value": "process_terminated", "override": true } }, + { "set": { "if": "ctx.event?.code == '6'", "field": "event.dataset", "value": "driver_loaded", "override": true } }, + { "set": { "if": "ctx.event?.code == '7'", "field": "event.dataset", "value": "image_loaded", "override": true } }, + { "set": { "if": "ctx.event?.code == '8'", "field": "event.dataset", "value": "create_remote_thread", "override": true } }, + { "set": { "if": "ctx.event?.code == '9'", "field": "event.dataset", "value": "raw_file_access_read", "override": true } }, + { "set": { "if": "ctx.event?.code == '10'", "field": "event.dataset", "value": "process_access", "override": true } }, + { "set": { "if": "ctx.event?.code == '11'", "field": "event.dataset", "value": "file_create", "override": true } }, + { "set": { "if": "ctx.event?.code == '12'", "field": "event.dataset", "value": "registry_create_delete", "override": true } }, + { "set": { "if": "ctx.event?.code == '13'", "field": "event.dataset", "value": "registry_value_set", "override": true } }, + { "set": { "if": "ctx.event?.code == '14'", "field": "event.dataset", "value": "registry_key_value_rename", "override": true } }, + { "set": { "if": "ctx.event?.code == '15'", "field": "event.dataset", "value": "file_create_stream_hash", "override": true } }, + { "set": { "if": "ctx.event?.code == '16'", "field": "event.dataset", "value": "config_change", "override": true } }, + { "set": { "if": "ctx.event?.code == '22'", "field": "event.dataset", "value": "dns_query", "override": true } }, + { "kv": {"field": "winlog.event_data.Hashes", "target_field": "file.hash", "field_split": ",", "value_split": "=", "ignore_missing": true } }, + { "kv": {"field": "winlog.event_data.Hash", "target_field": "file.hash", "field_split": ",", "value_split": "=", "ignore_missing": true } }, + { "rename": { "field": "file.hash.IMPHASH", "target_field": "hash.imphash", "ignore_missing":true } }, + { "rename": { "field": "file.hash.MD5", "target_field": "hash.md5", "ignore_missing":true } }, + { "rename": { "field": "file.hash.SHA256", "target_field": "hash.sha256", "ignore_missing":true } }, + { "rename": { "field": "winlog.event_data.SubjectUserName", "target_field": "user.name", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.DestinationHostname", "target_field": "destination.hostname", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.DestinationIp", "target_field": "destination.ip", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.DestinationPort", "target_field": "destination.port", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.image", "target_field": "process.executable", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.Image", "target_field": "process.executable", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.processID", "target_field": "process.pid", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ProcessId", "target_field": "process.pid", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.processGuid", "target_field": "process.entity_id", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ProcessGuid", "target_field": "process.entity_id", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.commandLine", "target_field": "process.command_line", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.CommandLine", "target_field": "process.command_line", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.currentDirectory", "target_field": "process.working_directory", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.CurrentDirectory", "target_field": "process.working_directory", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.description", "target_field": "process.pe.description", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.Description", "target_field": "process.pe.description", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.product", "target_field": "process.pe.product", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.Product", "target_field": "process.pe.product", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.company", "target_field": "process.pe.company", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.Company", "target_field": "process.pe.company", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.originalFileName", "target_field": "process.pe.original_file_name", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.OriginalFileName", "target_field": "process.pe.original_file_name", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.fileVersion", "target_field": "process.pe.file_version", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.FileVersion", "target_field": "process.pe.file_version", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.parentCommandLine", "target_field": "process.parent.command_line", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ParentCommandLine", "target_field": "process.parent.command_line", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.parentImage", "target_field": "process.parent.executable", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ParentImage", "target_field": "process.parent.executable", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.parentProcessGuid", "target_field": "process.parent.entity_id", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ParentProcessGuid", "target_field": "process.parent.entity_id", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.parentProcessId", "target_field": "process.ppid", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.ParentProcessId", "target_field": "process.ppid", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.Protocol", "target_field": "network.transport", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.User", "target_field": "user.name", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.SourceHostname", "target_field": "source.hostname", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.SourceIp", "target_field": "source.ip", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.SourcePort", "target_field": "source.port", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.targetFilename", "target_field": "file.target", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.TargetFilename", "target_field": "file.target", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.QueryResults", "target_field": "dns.answers.name", "ignore_missing": true } }, + { "rename": { "field": "winlog.event_data.QueryName", "target_field": "dns.query.name", "ignore_missing": true } }, + { "remove": { "field": "winlog.event_data.Hash", "ignore_missing": true } }, + { "remove": { "field": "winlog.event_data.Hashes", "ignore_missing": true } }, { "community_id": {} } ] } diff --git a/salt/elasticsearch/templates/component/so/so-scan-mappings.json b/salt/elasticsearch/templates/component/so/so-scan-mappings.json index 23e6142fc..87c959bfc 100644 --- a/salt/elasticsearch/templates/component/so/so-scan-mappings.json +++ b/salt/elasticsearch/templates/component/so/so-scan-mappings.json @@ -62,10 +62,40 @@ } } } - } + }, + "elf": { + "properties": { + "sections": { + "properties": { + "entropy": { + "type": "long" + } + } + } + } + } } } } } } } + + + + + + + + + + + + + + + + + + + diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7c0f78f96..055bd401b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -656,6 +656,49 @@ soc: - destination.geo.country_iso_code - user.name - source.ip + '::process_terminated': + - soc_timestamp + - process.executable + - process.pid + - winlog.computer_name + '::file_create': + - soc_timestamp + - file.target + - process.executable + - process.pid + - winlog.computer_name + '::registry_value_set': + - soc_timestamp + - winlog.event_data.TargetObject + - process.executable + - process.pid + - winlog.computer_name + '::process_creation': + - soc_timestamp + - process.command_line + - process.pid + - process.parent.executable + - process.working_directory + '::registry_create_delete': + - soc_timestamp + - winlog.event_data.TargetObject + - process.executable + - process.pid + - winlog.computer_name + '::dns_query': + - soc_timestamp + - dns.query.name + - dns.answers.name + - process.executable + - winlog.computer_name + '::file_create_stream_hash': + - soc_timestamp + - file.target + - hash.md5 + - hash.sha256 + - process.executable + - process.pid + - winlog.computer_name queryBaseFilter: queryToggleFilters: - name: caseExcludeToggle @@ -1373,6 +1416,49 @@ soc: - destination.geo.country_iso_code - user.name - source.ip + '::process_terminated': + - soc_timestamp + - process.executable + - process.pid + - winlog.computer_name + '::file_create': + - soc_timestamp + - file.target + - process.executable + - process.pid + - winlog.computer_name + '::registry_value_set': + - soc_timestamp + - winlog.event_data.TargetObject + - process.executable + - process.pid + - winlog.computer_name + '::process_creation': + - soc_timestamp + - process.command_line + - process.pid + - process.parent.executable + - process.working_directory + '::registry_create_delete': + - soc_timestamp + - winlog.event_data.TargetObject + - process.executable + - process.pid + - winlog.computer_name + '::dns_query': + - soc_timestamp + - dns.query.name + - dns.answers.name + - process.executable + - winlog.computer_name + '::file_create_stream_hash': + - soc_timestamp + - file.target + - hash.md5 + - hash.sha256 + - process.executable + - process.pid + - winlog.computer_name queryBaseFilter: queryToggleFilters: - name: caseExcludeToggle, From 79785fc0531fcfa739997d041152c769038432c2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 19 Sep 2022 15:26:32 -0400 Subject: [PATCH 04/21] zeek jinja --- salt/zeek/config.map.jinja | 7 ++++++ salt/zeek/files/node.cfg | 45 ---------------------------------- salt/zeek/files/node.cfg.jinja | 34 +++++++++++++++++++++++++ salt/zeek/init.sls | 9 ++++--- 4 files changed, 46 insertions(+), 49 deletions(-) delete mode 100644 salt/zeek/files/node.cfg create mode 100644 salt/zeek/files/node.cfg.jinja diff --git a/salt/zeek/config.map.jinja b/salt/zeek/config.map.jinja index a0f92463e..e321b3e3f 100644 --- a/salt/zeek/config.map.jinja +++ b/salt/zeek/config.map.jinja @@ -1,3 +1,10 @@ +{% import_yaml 'zeek/defaults.yaml' as zeek_defaults with context %} +{% set zeek_pillar = pillar.zeek %} +{% do ZEEKMERGED.zeek.config.node.update({'interface': pillar.sensor.interface})%} {# update this first so user can specify a differet interface with pillar.zeek.config.node.interface #} +{% set ZEEKMERGED = salt['defaults.merge'](zeek_defaults, zeek_pillar, in_place=False) %} + + + {% set ZEEKOPTIONS = {} %} {% set ENABLED = salt['pillar.get']('zeek:enabled', True) %} diff --git a/salt/zeek/files/node.cfg b/salt/zeek/files/node.cfg deleted file mode 100644 index 55f77982c..000000000 --- a/salt/zeek/files/node.cfg +++ /dev/null @@ -1,45 +0,0 @@ -{%- set interface = salt['pillar.get']('sensor:interface', 'bond0') %} -{%- if salt['pillar.get']('sensor:zeek_pins') or salt['pillar.get']('sensor:zeek_lbprocs') %} - {%- if salt['pillar.get']('sensor:zeek_proxies') %} - {%- set proxies = salt['pillar.get']('sensor:zeek_proxies', '1') %} - {%- else %} - {%- if salt['pillar.get']('sensor:zeek_pins') %} - {%- set proxies = (salt['pillar.get']('sensor:zeek_pins')|length/10)|round(0, 'ceil')|int %} - {%- else %} - {%- set proxies = (salt['pillar.get']('sensor:zeek_lbprocs')/10)|round(0, 'ceil')|int %} - {%- endif %} - {%- endif %} -[manager] -type=manager -host=localhost - -[logger] -type=logger -host=localhost - -[proxy] -type=proxy -host=localhost - -[worker-1] -type=worker -host=localhost -interface=af_packet::{{ interface }} -lb_method=custom - {%- if salt['pillar.get']('sensor:zeek_lbprocs') %} -lb_procs={{ salt['pillar.get']('sensor:zeek_lbprocs', '1') }} - {%- else %} -lb_procs={{ salt['pillar.get']('sensor:zeek_pins')|length }} - {%- endif %} - {%- if salt['pillar.get']('sensor:zeek_pins') %} -pin_cpus={{ salt['pillar.get']('sensor:zeek_pins')|join(", ") }} - {%- endif %} -af_packet_fanout_id=23 -af_packet_fanout_mode=AF_Packet::FANOUT_HASH -af_packet_buffer_size={{ salt['pillar.get']('sensor:zeek_buffer', 128*1024*1024) }} -{%- else %} -[zeeksa] -type=standalone -host=localhost -interface={{ interface }} -{%- endif %} diff --git a/salt/zeek/files/node.cfg.jinja b/salt/zeek/files/node.cfg.jinja new file mode 100644 index 000000000..201a2ab6e --- /dev/null +++ b/salt/zeek/files/node.cfg.jinja @@ -0,0 +1,34 @@ +[manager] +type=manager +host=localhost + +[logger] +type=logger +host=localhost + +[proxy] +type=proxy +host=localhost + +[worker-1] +type=worker +host=localhost +interface=af_packet::{{ ZEEKNODE.interface }} +lb_method=custom + {%- if ZEEKNODE.lbprocs %} +lb_procs={{ ZEEKNODE.lbprocs }} + {%- else %} +lb_procs={{ ZEEKNODE.zeek_pins | length }} + {%- endif %} + {%- if ZEEKNODE.zeek_pins %} +pin_cpus={{ ZEEKNODE.zeek_pins | join(", ") }} + {%- endif %} +af_packet_fanout_id=23 +af_packet_fanout_mode=AF_Packet::FANOUT_HASH +af_packet_buffer_size={{ ZEEKNODE.zeek_buffer }} +{%- else %} +[zeeksa] +type=standalone +host=localhost +interface={{ ZEEKNODE.interface }} +{%- endif %} diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index b6f3231ae..e5bc34716 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -7,6 +7,7 @@ {% if sls in allowed_states %} {% from "zeek/config.map.jinja" import ZEEKOPTIONS with context %} +{% from "zeek/config.map.jinja" import ZEEKMERGED with context %} {% set VERSION = salt['pillar.get']('global:soversion') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} @@ -15,8 +16,6 @@ {% set BPF_STATUS = 0 %} {% set INTERFACE = salt['pillar.get']('sensor:interface') %} -{% set ZEEK = salt['pillar.get']('zeek', {}) %} - # Zeek Salt State # Add Zeek group @@ -107,16 +106,18 @@ zeekctlcfg: - group: 939 - template: jinja - defaults: - ZEEKCTL: {{ ZEEK.zeekctl | tojson }} + ZEEKCTL: {{ ZEEKMERGED.zeek.config.zeekctl | tojson }} # Sync node.cfg nodecfg: file.managed: - name: /opt/so/conf/zeek/node.cfg - - source: salt://zeek/files/node.cfg + - source: salt://zeek/files/node.cfg,jinja - user: 937 - group: 939 - template: jinja + - defaults: + ZEEKNODE: {{ ZEEKMERGED.zeek.config.node }} networkscfg: file.managed: From ea7979cfdde0c20f433361d832eebdd1eed42d1c Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 19 Sep 2022 15:33:15 -0400 Subject: [PATCH 05/21] Add Elastic Agent datastreams to SOC index --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7c0f78f96..10ef4cb8d 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -64,7 +64,7 @@ soc: remoteHostUrls: [] username: password: - index: '*:so-*,*:endgame-*' + index: '*:so-*,*:endgame-*,.ds-logs*' cacheMs: 300000 verifyCert: false casesEnabled: true From 30afc88322fb2148214ea496a4e294764b0185e1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 19 Sep 2022 15:51:29 -0400 Subject: [PATCH 06/21] fix some soc defaults --- salt/soc/defaults.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 31a35a618..e25a25b5f 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -649,7 +649,7 @@ soc: queryBaseFilter: queryToggleFilters: - name: caseExcludeToggle - filter: NOT _index:\"*:so-case*\" + filter: 'NOT _index:"*:so-case*"' enabled: true queries: - name: Default Query @@ -1365,7 +1365,7 @@ soc: - source.ip queryBaseFilter: queryToggleFilters: - - name: caseExcludeToggle, + - name: caseExcludeToggle filter: 'NOT _index:"*:so-case*"' enabled: true queries: @@ -1591,7 +1591,7 @@ soc: - so_case.severity - so_case.assigneeId - so_case.createTime - queryBaseFilter: '_index:\"*:so-case\" AND so_kind:case' + queryBaseFilter: '_index:"*:so-case" AND so_kind:case' queryToggleFilters: [] queries: - name: Open Cases From 80919827c6bb45425005b07b052369f484fb49d0 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 19 Sep 2022 15:55:23 -0400 Subject: [PATCH 07/21] Fixup index patterns --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 10ef4cb8d..401d7fc21 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -64,7 +64,7 @@ soc: remoteHostUrls: [] username: password: - index: '*:so-*,*:endgame-*,.ds-logs*' + index: '*:so-*,*:endgame-*,*:logs-*' cacheMs: 300000 verifyCert: false casesEnabled: true From 509c32482f7f9ac69943b845252ebff11ab0f28d Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 19 Sep 2022 16:39:49 -0400 Subject: [PATCH 08/21] Update so-elasticsearch-templates-load to allow for proper loading of differently formatted Elastic Agent index templates --- salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load index cb727a5d3..aa9502396 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -58,7 +58,7 @@ echo cd ${ELASTICSEARCH_TEMPLATES}/index echo "Loading Security Onion index templates..." -for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; so-elasticsearch-query _index_template/so-$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done +for i in *; do TEMPLATE=${i::-14}; echo "$TEMPLATE"; so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done echo cd - >/dev/null From 3eb4adc5c373274f1efe4ee3a6958441a29c8a9a Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 19 Sep 2022 20:12:47 -0400 Subject: [PATCH 09/21] Hunt Query - Elastic Agent Live Osquery Logs --- salt/soc/defaults.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 9da87a898..7ba352e34 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -680,6 +680,9 @@ soc: - name: NIDS Alerts description: Show all NIDS alerts grouped by alert query: 'event.category: network AND event.dataset: alert | groupby rule.category rule.gid rule.uuid rule.name' + - name: Osquery - Live Query + description: Show all Osquery Live Query results + query: 'event.dataset: osquery_manager.result | groupby action_data.id action_data.query | groupby host.hostname' - name: Wazuh/OSSEC Alerts description: Show all Wazuh alerts at Level 5 or higher grouped by category query: 'event.module:ossec AND event.dataset:alert AND rule.level:>4 | groupby rule.category rule.name' From 120fdef17353e9a549585c1e67523dd21a279016 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 20 Sep 2022 08:27:47 -0400 Subject: [PATCH 10/21] Hunt Query - Elastic Agent Live Osquery Logs --- salt/soc/defaults.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7ba352e34..44e75feca 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -797,9 +797,6 @@ soc: - name: NTLM description: NTLM grouped by computer name query: 'event.dataset:ntlm | groupby ntlm.server.dns.name' - - name: Osquery Live Queries - description: Osquery Live Query results grouped by computer name - query: 'event.dataset:live_query | groupby host.hostname' - name: PE description: PE files list query: 'event.dataset:pe | groupby file.machine file.os file.subsystem' @@ -1457,9 +1454,6 @@ soc: - name: NTLM description: NTLM logs query: 'event.dataset:ntlm | groupby ntlm.server.dns.name | groupby ntlm.server.nb.name | groupby ntlm.server.tree.name | groupby ntlm.success | groupby source.ip | groupby destination.ip | groupby destination.port' - - name: Osquery Live Queries - description: Osquery Live Query results - query: 'event.dataset:live_query | groupby host.hostname' - name: PE description: PE files list query: 'event.dataset:pe | groupby file.machine | groupby file.os | groupby file.subsystem | groupby file.section_names | groupby file.is_exe | groupby file.is_64bit' From d1ee3a7d040cfa50b50d71da3b1d2ebb20d4fdd8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 20 Sep 2022 11:11:29 -0400 Subject: [PATCH 11/21] zeek 2.4 --- pillar/zeek/init.sls | 55 -------- salt/vars/sensor.map.jinja | 5 +- salt/zeek/config.map.jinja | 7 +- salt/zeek/defaults.yaml | 133 +++++++++--------- salt/zeek/files/networks.cfg.jinja | 12 +- salt/zeek/files/node.cfg.jinja | 16 +-- salt/zeek/init.sls | 22 +-- .../file-extraction/extract.zeek | 4 +- 8 files changed, 99 insertions(+), 155 deletions(-) delete mode 100644 pillar/zeek/init.sls diff --git a/pillar/zeek/init.sls b/pillar/zeek/init.sls deleted file mode 100644 index 5eeb273b9..000000000 --- a/pillar/zeek/init.sls +++ /dev/null @@ -1,55 +0,0 @@ -zeek: - zeekctl: - MailTo: root@localhost - MailConnectionSummary: 1 - MinDiskSpace: 5 - MailHostUpDown: 1 - LogRotationInterval: 3600 - LogExpireInterval: 0 - StatsLogEnable: 1 - StatsLogExpireInterval: 0 - StatusCmdShowAll: 0 - CrashExpireInterval: 0 - SitePolicyScripts: local.zeek - LogDir: /nsm/zeek/logs - SpoolDir: /nsm/zeek/spool - CfgDir: /opt/zeek/etc - CompressLogs: 1 - local: - '@load': - - misc/loaded-scripts - - tuning/defaults - - misc/capture-loss - - misc/stats - - frameworks/software/vulnerable - - frameworks/software/version-changes - - protocols/ftp/software - - protocols/smtp/software - - protocols/ssh/software - - protocols/http/software - - protocols/dns/detect-external-names - - protocols/ftp/detect - - protocols/conn/known-hosts - - protocols/conn/known-services - - protocols/ssl/known-certs - - protocols/ssl/validate-certs - - protocols/ssl/log-hostcerts-only - - protocols/ssh/geo-data - - protocols/ssh/detect-bruteforcing - - protocols/ssh/interesting-hostnames - - protocols/http/detect-sqli - - frameworks/files/hash-all-files - - frameworks/files/detect-MHR - - policy/frameworks/notice/extend-email/hostnames - - ja3 - - hassh - - intel - - cve-2020-0601 - - securityonion/bpfconf - - securityonion/communityid - - securityonion/file-extraction - '@load-sigs': - - frameworks/signatures/detect-windows-shells - redef: - - LogAscii::use_json = T; - - CaptureLoss::watch_interval = 5 mins; diff --git a/salt/vars/sensor.map.jinja b/salt/vars/sensor.map.jinja index 477761d7c..33f26de84 100644 --- a/salt/vars/sensor.map.jinja +++ b/salt/vars/sensor.map.jinja @@ -1,6 +1,9 @@ {% set ROLE_GLOBALS = {} %} -{% set SENSOR_GLOBALS = [] +{% set SENSOR_GLOBALS = { + 'sensor': { + 'interface': INIT.PILLAR.sensor.interface + } %} {% for sg in SENSOR_GLOBALS %} diff --git a/salt/zeek/config.map.jinja b/salt/zeek/config.map.jinja index e321b3e3f..74e4942c2 100644 --- a/salt/zeek/config.map.jinja +++ b/salt/zeek/config.map.jinja @@ -1,10 +1,9 @@ +{% from 'vars/sensor.map.jinja' import GLOBALS %} {% import_yaml 'zeek/defaults.yaml' as zeek_defaults with context %} -{% set zeek_pillar = pillar.zeek %} -{% do ZEEKMERGED.zeek.config.node.update({'interface': pillar.sensor.interface})%} {# update this first so user can specify a differet interface with pillar.zeek.config.node.interface #} +{% set zeek_pillar = salt['pillar.get']('zeek', []) %} +{% do ZEEKMERGED.zeek.config.node.update({'interface': GLOBALS.sensor.interface}) %} {# update this first so user can specify a differet interface with pillar.zeek.config.node.interface #} {% set ZEEKMERGED = salt['defaults.merge'](zeek_defaults, zeek_pillar, in_place=False) %} - - {% set ZEEKOPTIONS = {} %} {% set ENABLED = salt['pillar.get']('zeek:enabled', True) %} diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml index 8d2a96444..b3cd183cd 100644 --- a/salt/zeek/defaults.yaml +++ b/salt/zeek/defaults.yaml @@ -20,68 +20,71 @@ zeek: SpoolDir: /nsm/zeek/spool CfgDir: /opt/zeek/etc CompressLogs: 1 - policy: - file_extraction: - - application/x-dosexec: exe - - application/pdf: pdf - - application/msword: doc - - application/vnd.ms-powerpoint: doc - - application/rtf: doc - - application/vnd.ms-word.document.macroenabled.12: doc - - application/vnd.ms-word.template.macroenabled.12: doc - - application/vnd.ms-powerpoint.template.macroenabled.12: doc - - application/vnd.ms-excel: doc - - application/vnd.ms-excel.addin.macroenabled.12: doc - - application/vnd.ms-excel.sheet.binary.macroenabled.12: doc - - application/vnd.ms-excel.template.macroenabled.12: doc - - application/vnd.ms-excel.sheet.macroenabled.12: doc - - application/vnd.openxmlformats-officedocument.presentationml.presentation: doc - - application/vnd.openxmlformats-officedocument.presentationml.slide: doc - - application/vnd.openxmlformats-officedocument.presentationml.slideshow: doc - - application/vnd.openxmlformats-officedocument.presentationml.template: doc - - application/vnd.openxmlformats-officedocument.spreadsheetml.sheet: doc - - application/vnd.openxmlformats-officedocument.spreadsheetml.template: doc - - application/vnd.openxmlformats-officedocument.wordprocessingml.document: doc - - application/vnd.openxmlformats-officedocument.wordprocessingml.template: doc - - application/vnd.ms-powerpoint.addin.macroenabled.12: doc - - application/vnd.ms-powerpoint.slide.macroenabled.12: doc - - application/vnd.ms-powerpoint.presentation.macroenabled.12: doc - - application/vnd.ms-powerpoint.slideshow.macroenabled.12: doc - - application/vnd.openxmlformats-officedocument: doc - load: - - misc/loaded-scripts - - tuning/defaults - - misc/capture-loss - - misc/stats - - frameworks/software/vulnerable - - frameworks/software/version-changes - - protocols/ftp/software - - protocols/smtp/software - - protocols/ssh/software - - protocols/http/software - - protocols/dns/detect-external-names - - protocols/ftp/detect - - protocols/conn/known-hosts - - protocols/conn/known-services - - protocols/ssl/known-certs - - protocols/ssl/validate-certs - - protocols/ssl/log-hostcerts-only - - protocols/ssh/geo-data - - protocols/ssh/detect-bruteforcing - - protocols/ssh/interesting-hostnames - - protocols/http/detect-sqli - - frameworks/files/hash-all-files - - frameworks/files/detect-MHR - - policy/frameworks/notice/extend-email/hostnames - - ja3 - - hassh - - intel - - cve-2020-0601 - - securityonion/bpfconf - - securityonion/communityid - - securityonion/file-extraction - load-sigs: - - frameworks/signatures/detect-windows-shells - redef: - - LogAscii::use_json = T; - - CaptureLoss::watch_interval = 5 mins; \ No newline at end of file + local: + load: + - misc/loaded-scripts + - tuning/defaults + - misc/capture-loss + - misc/stats + - frameworks/software/vulnerable + - frameworks/software/version-changes + - protocols/ftp/software + - protocols/smtp/software + - protocols/ssh/software + - protocols/http/software + - protocols/dns/detect-external-names + - protocols/ftp/detect + - protocols/conn/known-hosts + - protocols/conn/known-services + - protocols/ssl/known-certs + - protocols/ssl/validate-certs + - protocols/ssl/log-hostcerts-only + - protocols/ssh/geo-data + - protocols/ssh/detect-bruteforcing + - protocols/ssh/interesting-hostnames + - protocols/http/detect-sqli + - frameworks/files/hash-all-files + - frameworks/files/detect-MHR + - policy/frameworks/notice/extend-email/hostnames + - ja3 + - hassh + - intel + - cve-2020-0601 + - securityonion/bpfconf + - securityonion/communityid + - securityonion/file-extraction + load-sigs: + - frameworks/signatures/detect-windows-shells + redef: + - LogAscii::use_json = T; + - CaptureLoss::watch_interval = 5 mins; + networks: + HOME_NET: "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" + file_extraction: + - application/x-dosexec: exe + - application/pdf: pdf + - application/msword: doc + - application/vnd.ms-powerpoint: doc + - application/rtf: doc + - application/vnd.ms-word.document.macroenabled.12: doc + - application/vnd.ms-word.template.macroenabled.12: doc + - application/vnd.ms-powerpoint.template.macroenabled.12: doc + - application/vnd.ms-excel: doc + - application/vnd.ms-excel.addin.macroenabled.12: doc + - application/vnd.ms-excel.sheet.binary.macroenabled.12: doc + - application/vnd.ms-excel.template.macroenabled.12: doc + - application/vnd.ms-excel.sheet.macroenabled.12: doc + - application/vnd.openxmlformats-officedocument.presentationml.presentation: doc + - application/vnd.openxmlformats-officedocument.presentationml.slide: doc + - application/vnd.openxmlformats-officedocument.presentationml.slideshow: doc + - application/vnd.openxmlformats-officedocument.presentationml.template: doc + - application/vnd.openxmlformats-officedocument.spreadsheetml.sheet: doc + - application/vnd.openxmlformats-officedocument.spreadsheetml.template: doc + - application/vnd.openxmlformats-officedocument.wordprocessingml.document: doc + - application/vnd.openxmlformats-officedocument.wordprocessingml.template: doc + - application/vnd.ms-powerpoint.addin.macroenabled.12: doc + - application/vnd.ms-powerpoint.slide.macroenabled.12: doc + - application/vnd.ms-powerpoint.presentation.macroenabled.12: doc + - application/vnd.ms-powerpoint.slideshow.macroenabled.12: doc + - application/vnd.openxmlformats-officedocument: doc + bpf: [] diff --git a/salt/zeek/files/networks.cfg.jinja b/salt/zeek/files/networks.cfg.jinja index f1ee065de..5818380ce 100644 --- a/salt/zeek/files/networks.cfg.jinja +++ b/salt/zeek/files/networks.cfg.jinja @@ -1,9 +1,5 @@ -{%- if salt['pillar.get']('sensor:hnsensor') %} -{%- set HOME_NET = salt['pillar.get']('sensor:hnsensor') %} -{%- else %} -{%- set HOME_NET = salt['pillar.get']('global:hnmanager') %} -{%- endif %} -{%- set HNLIST = HOME_NET.split(',') %} -{%- for HN in HNLIST %} +{%- if NETWORKS.HOME_NET %} +{%- for HN in NETWORKS.HOME_NET.split(',') %} {{ HN }} -{%- endfor %} +{%- endfor %} +{%- endif %} diff --git a/salt/zeek/files/node.cfg.jinja b/salt/zeek/files/node.cfg.jinja index 201a2ab6e..f852e2ef0 100644 --- a/salt/zeek/files/node.cfg.jinja +++ b/salt/zeek/files/node.cfg.jinja @@ -13,22 +13,22 @@ host=localhost [worker-1] type=worker host=localhost -interface=af_packet::{{ ZEEKNODE.interface }} +interface=af_packet::{{ NODE.interface }} lb_method=custom - {%- if ZEEKNODE.lbprocs %} -lb_procs={{ ZEEKNODE.lbprocs }} + {%- if NODE.lbprocs %} +lb_procs={{ NODE.lbprocs }} {%- else %} -lb_procs={{ ZEEKNODE.zeek_pins | length }} +lb_procs={{ NODE.zeek_pins | length }} {%- endif %} - {%- if ZEEKNODE.zeek_pins %} -pin_cpus={{ ZEEKNODE.zeek_pins | join(", ") }} + {%- if NODE.zeek_pins %} +pin_cpus={{ NODE.zeek_pins | join(", ") }} {%- endif %} af_packet_fanout_id=23 af_packet_fanout_mode=AF_Packet::FANOUT_HASH -af_packet_buffer_size={{ ZEEKNODE.zeek_buffer }} +af_packet_buffer_size={{ NODE.zeek_buffer }} {%- else %} [zeeksa] type=standalone host=localhost -interface={{ ZEEKNODE.interface }} +interface={{ NODE.interface }} {%- endif %} diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index e5bc34716..c2b1af5d0 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -6,15 +6,11 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS with context %} {% from "zeek/config.map.jinja" import ZEEKOPTIONS with context %} {% from "zeek/config.map.jinja" import ZEEKMERGED with context %} -{% set VERSION = salt['pillar.get']('global:soversion') %} -{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} -{% set MANAGER = salt['grains.get']('master') %} -{% set BPF_ZEEK = salt['pillar.get']('zeek:bpf', {}) %} {% set BPF_STATUS = 0 %} -{% set INTERFACE = salt['pillar.get']('sensor:interface') %} # Zeek Salt State @@ -76,6 +72,8 @@ zeekpolicysync: - user: 937 - group: 939 - template: jinja + - defaults: + FILE_EXTRACTION: {{ ZEEKMERGED.file_extraction }} # Ensure the zeek spool tree (and state.db) ownership is correct zeekspoolownership: @@ -117,7 +115,7 @@ nodecfg: - group: 939 - template: jinja - defaults: - ZEEKNODE: {{ ZEEKMERGED.zeek.config.node }} + NODE: {{ ZEEKMERGED.zeek.config.node }} networkscfg: file.managed: @@ -126,6 +124,8 @@ networkscfg: - user: 937 - group: 939 - template: jinja + - defaults: + NETWORKS: {{ ZEEKMERGED.zeek.networks }} #zeekcleanscript: # file.managed: @@ -159,8 +159,8 @@ zeekpacketlosscron: - dayweek: '*' # BPF compilation and configuration -{% if BPF_ZEEK %} - {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', INTERFACE + ' ' + BPF_ZEEK|join(" "),cwd='/root') %} +{% if ZEEKMERGED.zeek.bpf %} + {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + ZEEKMERGED.zeek.bpf|join(" "),cwd='/root') %} {% if BPF_CALC['stderr'] == "" %} {% set BPF_STATUS = 1 %} {% else %} @@ -178,7 +178,7 @@ zeekbpf: - user: 940 - group: 940 {% if BPF_STATUS %} - - contents_pillar: zeek:bpf + - contents: {{ ZEEKMERGED.bpf }} {% else %} - contents: - "ip or not ip" @@ -193,12 +193,12 @@ localzeek: - group: 939 - template: jinja - defaults: - LOCAL: {{ ZEEK.local | tojson }} + LOCAL: {{ ZEEKMERGED.zeek.config.local | tojson }} so-zeek: docker_container.{{ ZEEKOPTIONS.status }}: {% if ZEEKOPTIONS.status == 'running' %} - - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }} + - image: {{ GLOBALS.manager }}:5000/{{ GLOBALS.image_repo }}/so-zeek:{{ GLOBALS.so_version }} - start: {{ ZEEKOPTIONS.start }} - privileged: True - ulimits: diff --git a/salt/zeek/policy/securityonion/file-extraction/extract.zeek b/salt/zeek/policy/securityonion/file-extraction/extract.zeek index 8cdaf42dd..d4ba0551e 100644 --- a/salt/zeek/policy/securityonion/file-extraction/extract.zeek +++ b/salt/zeek/policy/securityonion/file-extraction/extract.zeek @@ -1,5 +1,3 @@ -{% import_yaml "zeek/fileextraction_defaults.yaml" as zeek_default -%} -{% set zeek = salt['grains.filter_by'](zeek_default, default='zeek', merge=salt['pillar.get']('zeek', {})) -%} # Directory to stage Zeek extracted files before processing redef FileExtract::prefix = "/nsm/zeek/extracted/"; # Set a limit to the file size @@ -7,7 +5,7 @@ redef FileExtract::default_limit = 9000000; # These are the mimetypes we want to rip off the networks export { global _mime_whitelist: table[string] of string = { - {%- for li in zeek.policy.file_extraction %} + {%- for li in FILE_EXTRACTION %} {%- if not loop.last %} {%- for k,v in li.items() %} ["{{ k }}"] = "{{ v }}", From aadce055d1cf1f1bcbfd34f817bc062b5106b54e Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 20 Sep 2022 12:49:14 -0400 Subject: [PATCH 12/21] change version to 2.4.0 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0662e05be..2d3442035 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -## Security Onion 2.3.120 +## Security Onion 2.4.0 -Security Onion 2.3.120 is here! +Security Onion 2.4.0 is here! ## Screenshots From 75aa121b2d29fa9298a5d43b131c4d1ffc48e508 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 20 Sep 2022 13:19:15 -0400 Subject: [PATCH 13/21] fix some things --- pillar/zeek/init.sls | 1 + salt/vars/sensor.map.jinja | 7 +++---- salt/zeek/config.map.jinja | 4 ++-- salt/zeek/defaults.yaml | 2 +- salt/zeek/init.sls | 6 +++--- 5 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 pillar/zeek/init.sls diff --git a/pillar/zeek/init.sls b/pillar/zeek/init.sls new file mode 100644 index 000000000..c06759744 --- /dev/null +++ b/pillar/zeek/init.sls @@ -0,0 +1 @@ +zeek: diff --git a/salt/vars/sensor.map.jinja b/salt/vars/sensor.map.jinja index 33f26de84..425d72969 100644 --- a/salt/vars/sensor.map.jinja +++ b/salt/vars/sensor.map.jinja @@ -2,10 +2,9 @@ {% set SENSOR_GLOBALS = { 'sensor': { - 'interface': INIT.PILLAR.sensor.interface + 'interface': pillar.sensor.interface + } } %} -{% for sg in SENSOR_GLOBALS %} -{% do salt['defaults.merge'](ROLE_GLOBALS, sg, merge_lists=False, in_place=True) %} -{% endfor %} +{% do salt['defaults.merge'](ROLE_GLOBALS, SENSOR_GLOBALS, merge_lists=False, in_place=True) %} diff --git a/salt/zeek/config.map.jinja b/salt/zeek/config.map.jinja index 74e4942c2..ced2175e7 100644 --- a/salt/zeek/config.map.jinja +++ b/salt/zeek/config.map.jinja @@ -1,8 +1,8 @@ -{% from 'vars/sensor.map.jinja' import GLOBALS %} +{% from 'vars/sensor.map.jinja' import ROLE_GLOBALS %} {% import_yaml 'zeek/defaults.yaml' as zeek_defaults with context %} {% set zeek_pillar = salt['pillar.get']('zeek', []) %} -{% do ZEEKMERGED.zeek.config.node.update({'interface': GLOBALS.sensor.interface}) %} {# update this first so user can specify a differet interface with pillar.zeek.config.node.interface #} {% set ZEEKMERGED = salt['defaults.merge'](zeek_defaults, zeek_pillar, in_place=False) %} +{% do ZEEKMERGED.zeek.config.node.update({'interface': ROLE_GLOBALS.sensor.interface}) %} {% set ZEEKOPTIONS = {} %} {% set ENABLED = salt['pillar.get']('zeek:enabled', True) %} diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml index b3cd183cd..eb7ce8453 100644 --- a/salt/zeek/defaults.yaml +++ b/salt/zeek/defaults.yaml @@ -59,7 +59,7 @@ zeek: - LogAscii::use_json = T; - CaptureLoss::watch_interval = 5 mins; networks: - HOME_NET: "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" + HOME_NET: 192.168.0.0/16,10.0.0.0/8,172.16.0.0/12 file_extraction: - application/x-dosexec: exe - application/pdf: pdf diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index c2b1af5d0..b292a20a3 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -73,7 +73,7 @@ zeekpolicysync: - group: 939 - template: jinja - defaults: - FILE_EXTRACTION: {{ ZEEKMERGED.file_extraction }} + FILE_EXTRACTION: {{ ZEEKMERGED.zeek.file_extraction }} # Ensure the zeek spool tree (and state.db) ownership is correct zeekspoolownership: @@ -110,7 +110,7 @@ zeekctlcfg: nodecfg: file.managed: - name: /opt/so/conf/zeek/node.cfg - - source: salt://zeek/files/node.cfg,jinja + - source: salt://zeek/files/node.cfg.jinja - user: 937 - group: 939 - template: jinja @@ -125,7 +125,7 @@ networkscfg: - group: 939 - template: jinja - defaults: - NETWORKS: {{ ZEEKMERGED.zeek.networks }} + NETWORKS: {{ ZEEKMERGED.zeek.config.networks }} #zeekcleanscript: # file.managed: From 0eafed32a4b76ac6b2179ce840dabf7e9657ad7c Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 20 Sep 2022 15:13:14 -0400 Subject: [PATCH 14/21] Update docs links in README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2d3442035..e6d616d05 100644 --- a/README.md +++ b/README.md @@ -12,24 +12,24 @@ Hunt ### Release Notes -https://docs.securityonion.net/en/2.3/release-notes.html +https://docs.securityonion.net/en/2.4/release-notes.html ### Requirements -https://docs.securityonion.net/en/2.3/hardware.html +https://docs.securityonion.net/en/2.4/hardware.html ### Download -https://docs.securityonion.net/en/2.3/download.html +https://docs.securityonion.net/en/2.4/download.html ### Installation -https://docs.securityonion.net/en/2.3/installation.html +https://docs.securityonion.net/en/2.4/installation.html ### FAQ -https://docs.securityonion.net/en/2.3/faq.html +https://docs.securityonion.net/en/2.4/faq.html ### Feedback -https://docs.securityonion.net/en/2.3/community-support.html +https://docs.securityonion.net/en/2.4/community-support.html From 530c4978002717b89fad989d61cca887fb065a6a Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 20 Sep 2022 15:16:04 -0400 Subject: [PATCH 15/21] Update motd.md --- salt/soc/files/soc/motd.md | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/salt/soc/files/soc/motd.md b/salt/soc/files/soc/motd.md index fbc643993..40964349d 100644 --- a/salt/soc/files/soc/motd.md +++ b/salt/soc/files/soc/motd.md @@ -6,22 +6,8 @@ If you're ready to dive in, take a look at the [Alerts](/#/alerts) interface to ## What's New -To see all the latest features and fixes in this version of Security Onion, click the upper-right menu and then click the [What's New](/docs/#release-notes) link. +To see all the latest features and fixes in this version of Security Onion, click the upper-right menu and then click the [What's New](/docs/release-notes.html) link. ## Customize This Space -Make this area your own by customizing the content. The content is stored in the `motd.md` file, which uses the common Markdown (.md) format. To learn more about the format, please see [markdownguide.org](https://www.markdownguide.org/). - -To customize this content, login to the manager via SSH and execute the following command: - -```bash -sudo cp /opt/so/saltstack/default/salt/soc/files/soc/motd.md /opt/so/saltstack/local/salt/soc/files/soc/ -``` - -Then edit the new file as desired using your favorite text editor. - -Finally, restart SOC to make the changes take effect: - -```bash -sudo so-soc-restart -``` +Make this area your own by customizing the content in the [Config](/#/config) interface. From 0137004344ab05b05664d51aac43ce8ada0043be Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 20 Sep 2022 15:16:53 -0400 Subject: [PATCH 16/21] Fix releaseNotesUrl in defaults.yaml --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 041a09d0b..828e90dda 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -93,7 +93,7 @@ soc: client: docsUrl: /docs/ cheatsheetUrl: /docs/cheatsheet.pdf - releaseNotesUrl: /docs/#release-notes + releaseNotesUrl: /docs/release-notes.html apiTimeoutMs: 0 webSocketTimeoutMs: 0 tipTimeoutMs: 0 From 1685e0e6db563a7374eee749dc486e5362cbf8e3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 20 Sep 2022 15:25:50 -0400 Subject: [PATCH 17/21] few more --- salt/zeek/defaults.yaml | 11 ++++++----- salt/zeek/files/node.cfg.jinja | 13 +++++++------ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml index eb7ce8453..681f29df0 100644 --- a/salt/zeek/defaults.yaml +++ b/salt/zeek/defaults.yaml @@ -1,9 +1,10 @@ zeek: config: node: - lb_procs: 1 - zeek_pins_enabled: False - zeek_pins: [] + lb_procs: 0 + pins_enabled: False + pins: [] + buffer: 128*1024*1024 zeekctl: MailTo: root@localhost MailConnectionSummary: 1 @@ -21,7 +22,7 @@ zeek: CfgDir: /opt/zeek/etc CompressLogs: 1 local: - load: + '@load': - misc/loaded-scripts - tuning/defaults - misc/capture-loss @@ -53,7 +54,7 @@ zeek: - securityonion/bpfconf - securityonion/communityid - securityonion/file-extraction - load-sigs: + '@load-sigs': - frameworks/signatures/detect-windows-shells redef: - LogAscii::use_json = T; diff --git a/salt/zeek/files/node.cfg.jinja b/salt/zeek/files/node.cfg.jinja index f852e2ef0..9ef06bd5a 100644 --- a/salt/zeek/files/node.cfg.jinja +++ b/salt/zeek/files/node.cfg.jinja @@ -1,3 +1,4 @@ +{%- if NODE.pins or NODE.lb_procs %} [manager] type=manager host=localhost @@ -15,17 +16,17 @@ type=worker host=localhost interface=af_packet::{{ NODE.interface }} lb_method=custom - {%- if NODE.lbprocs %} -lb_procs={{ NODE.lbprocs }} + {%- if NODE.lb_procs %} +lb_procs={{ NODE.lb_procs }} {%- else %} -lb_procs={{ NODE.zeek_pins | length }} +lb_procs={{ NODE.pins | length }} {%- endif %} - {%- if NODE.zeek_pins %} -pin_cpus={{ NODE.zeek_pins | join(", ") }} + {%- if NODE.pins %} +pin_cpus={{ NODE.pins | join(", ") }} {%- endif %} af_packet_fanout_id=23 af_packet_fanout_mode=AF_Packet::FANOUT_HASH -af_packet_buffer_size={{ NODE.zeek_buffer }} +af_packet_buffer_size={{ NODE.buffer }} {%- else %} [zeeksa] type=standalone From e1ea3c20315ffd07b816480009aad29cc779efd6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 20 Sep 2022 16:22:54 -0400 Subject: [PATCH 18/21] soc for zeek --- salt/zeek/soc_zeek.yaml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/salt/zeek/soc_zeek.yaml b/salt/zeek/soc_zeek.yaml index a48ec20dc..7da21aa41 100644 --- a/salt/zeek/soc_zeek.yaml +++ b/salt/zeek/soc_zeek.yaml @@ -3,16 +3,23 @@ zeek: enabled: description: This is a list of zeek logs that will be shipped through the pipeline. If you remove a log from this list it will still persist on the sensor. config: + local: + '@load': + description: List of Zeek policies to load + '@load-sigs': + description: List of Zeek signatures to load node: lb_procs: description: This is the amount of CPUs to use for Zeek. This setting is ignored if you are using pins. node: True - zeek_pins_enabled: - description: + pins_enabled: + description: Enabled CPU pinning node: True - zeek_pins: - description: List of CPUs you want to + advanced: True + pins: + description: List of CPUs you want to pin to node: True + advanced: True zeekctl: CompressLogs: description: Enable compression of zeek logs. If you are seeing packet loss at the top of the hour in zeek or pcap you might need to set this to 0. This will use more disk space but save IO and CPU. @@ -24,10 +31,6 @@ zeek: file: True global: True advanced: True - file_extraction: - description: This is a list of mime types Zeek will extract from the network streams. - load: - description: List of Zeek policies to load - load-sigs: - description: List of Zeek signatures to load + file_extraction: + description: This is a list of mime types Zeek will extract from the network streams. \ No newline at end of file From 7f2c5bc7572131ef0d5335d331cadbd6b0377b9b Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 20 Sep 2022 20:27:26 +0000 Subject: [PATCH 19/21] Add component templates for Fleet --- .../so-fleet_agent_id_verification-1.json | 36 +++++++++++++++++++ .../elastic-agent/so-fleet_globals-1.json | 34 ++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json create mode 100644 salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json new file mode 100644 index 000000000..e3b768ae3 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json @@ -0,0 +1,36 @@ +{ + "component_templates": [ + { + "name": ".fleet_agent_id_verification-1", + "component_template": { + "template": { + "settings": { + "index": { + "final_pipeline": ".fleet_final_pipeline-1" + } + }, + "mappings": { + "properties": { + "event": { + "properties": { + "agent_id_status": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingested": { + "format": "strict_date_time_no_millis||strict_date_optional_time||epoch_millis", + "type": "date" + } + } + } + } + } + }, + "_meta": { + "managed_by": "fleet", + "managed": true + } + } + } + ] +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json new file mode 100644 index 000000000..002529d01 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json @@ -0,0 +1,34 @@ +{ + "component_templates": [ + { + "name": ".fleet_globals-1", + "component_template": { + "template": { + "settings": {}, + "mappings": { + "_meta": { + "managed_by": "fleet", + "managed": true + }, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "date_detection": false + } + }, + "_meta": { + "managed_by": "fleet", + "managed": true + } + } + } + ] +} From 46dd4c2749e376c94d3827aad8f5a24e8ebd7ce8 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 20 Sep 2022 20:33:06 +0000 Subject: [PATCH 20/21] Rename component mappings and references for Security Onion --- salt/elasticsearch/defaults.yaml | 44 ++++++++--------- .../so-fleet_agent_id_verification-1.json | 40 +++++++++++++++- .../elastic-agent/so-fleet_globals-1.json | 47 +++++++++++++++++-- 3 files changed, 104 insertions(+), 27 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index a14c03e2d..37eab28a0 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -80,8 +80,8 @@ elasticsearch: composed_of: - "so-logs-elastic_agent.apm_server@package" - "so-logs-elastic_agent.apm_server@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" priority: 500 _meta: package: @@ -114,8 +114,8 @@ elasticsearch: composed_of: - "so-logs-elastic_agent.auditbeat@package" - "so-logs-elastic_agent.auditbeat@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" priority: 500 _meta: package: @@ -148,8 +148,8 @@ elasticsearch: composed_of: - "so-logs-elastic_agent.cloudbeat@package" - "so-logs-elastic_agent.cloudbeat@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" priority: 500 _meta: package: @@ -182,8 +182,8 @@ elasticsearch: composed_of: - "so-logs-elastic_agent.endpoint_security@package" - "so-logs-elastic_agent.endpoint_security@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" priority: 500 _meta: package: @@ -216,8 +216,8 @@ elasticsearch: composed_of: - "so-logs-elastic_agent.filebeat@package" - "so-logs-elastic_agent.filebeat@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" priority: 500 _meta: package: @@ -250,8 +250,8 @@ elasticsearch: composed_of: - "so-logs-elastic_agent.fleet_server@package" - "so-logs-elastic_agent.fleet_server@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" priority: 500 _meta: package: @@ -284,8 +284,8 @@ elasticsearch: composed_of: - "so-logs-elastic_agent.heartbeat@package" - "so-logs-elastic_agent.heartbeat@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" priority: 500 _meta: package: @@ -318,8 +318,8 @@ elasticsearch: composed_of: - "so-logs-elastic_agent@package" - "so-logs-elastic_agent@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" priority: 500 _meta: package: @@ -352,8 +352,8 @@ elasticsearch: composed_of: - "so-logs-elastic_agent.metricbeat@package" - "so-logs-elastic_agent.metricbeat@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" priority: 500 _meta: package: @@ -386,8 +386,8 @@ elasticsearch: composed_of: - "so-logs-elastic_agent.osquerybeat@package" - "so-logs-elastic_agent.osquerybeat@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" priority: 500 _meta: package: @@ -420,8 +420,8 @@ elasticsearch: composed_of: - "so-logs-elastic_agent.packetbeat@package" - "so-logs-elastic_agent.packetbeat@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" priority: 500 _meta: package: diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json index e3b768ae3..cac2cd8ee 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json +++ b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json @@ -1,10 +1,48 @@ { "component_templates": [ { - "name": ".fleet_agent_id_verification-1", + "name": "so-fleet_agent_id_verification-1", "component_template": { "template": { "settings": { + "analysis": { + "analyzer": { + "es_security_analyzer": { + "type": "custom", + "char_filter": [ + "whitespace_no_way" + ], + "filter": [ + "lowercase", + "trim" + ], + "tokenizer": "keyword" + } + }, + "char_filter": { + "whitespace_no_way": { + "type": "pattern_replace", + "pattern": "(\\s)+", + "replacement": "$1" + } + }, + "filter": { + "path_hierarchy_pattern_filter": { + "type": "pattern_capture", + "preserve_original": true, + "patterns": [ + "((?:[^\\\\]*\\\\)*)(.*)", + "((?:[^/]*/)*)(.*)" + ] + } + }, + "tokenizer": { + "path_tokenizer": { + "type": "path_hierarchy", + "delimiter": "\\" + } + } + }, "index": { "final_pipeline": ".fleet_final_pipeline-1" } diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json index 002529d01..5e569846c 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json +++ b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json @@ -1,13 +1,52 @@ { "component_templates": [ { - "name": ".fleet_globals-1", + "name": "so-fleet_globals-1", "component_template": { "template": { - "settings": {}, + "settings": { + "analysis": { + "analyzer": { + "es_security_analyzer": { + "type": "custom", + "char_filter": [ + "whitespace_no_way" + ], + "filter": [ + "lowercase", + "trim" + ], + "tokenizer": "keyword" + } + }, + "char_filter": { + "whitespace_no_way": { + "type": "pattern_replace", + "pattern": "(\\s)+", + "replacement": "$1" + } + }, + "filter": { + "path_hierarchy_pattern_filter": { + "type": "pattern_capture", + "preserve_original": true, + "patterns": [ + "((?:[^\\\\]*\\\\)*)(.*)", + "((?:[^/]*/)*)(.*)" + ] + } + }, + "tokenizer": { + "path_tokenizer": { + "type": "path_hierarchy", + "delimiter": "\\" + } + } + } + }, "mappings": { "_meta": { - "managed_by": "fleet", + "managed_by": "security_onion", "managed": true }, "dynamic_templates": [ @@ -25,7 +64,7 @@ } }, "_meta": { - "managed_by": "fleet", + "managed_by": "security_onion", "managed": true } } From 2995ae32bd300acfc5e9f4727a64cdace33811e8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 22 Sep 2022 10:49:26 -0400 Subject: [PATCH 21/21] 2.4 fw changes --- files/firewall/portgroups.local.yaml | 3 +- salt/firewall/assigned_hostgroups.map.yaml | 40 +++--- salt/firewall/hostgroups/beats_endpoint | 0 salt/firewall/hostgroups/beats_endpoint_ssl | 0 .../hostgroups/elastic_agent_endpoint | 0 salt/firewall/hostgroups/elasticsearch_rest | 0 salt/firewall/hostgroups/endgame | 0 salt/firewall/hostgroups/minion | 0 salt/firewall/hostgroups/strelka_frontend | 0 salt/firewall/hostgroups/syslog | 0 salt/firewall/init.sls | 2 +- salt/firewall/map.jinja | 33 ++++- salt/firewall/portgroups.yaml | 116 ------------------ 13 files changed, 52 insertions(+), 142 deletions(-) create mode 100644 salt/firewall/hostgroups/beats_endpoint create mode 100644 salt/firewall/hostgroups/beats_endpoint_ssl create mode 100644 salt/firewall/hostgroups/elastic_agent_endpoint create mode 100644 salt/firewall/hostgroups/elasticsearch_rest create mode 100644 salt/firewall/hostgroups/endgame create mode 100644 salt/firewall/hostgroups/minion create mode 100644 salt/firewall/hostgroups/strelka_frontend create mode 100644 salt/firewall/hostgroups/syslog delete mode 100644 salt/firewall/portgroups.yaml diff --git a/files/firewall/portgroups.local.yaml b/files/firewall/portgroups.local.yaml index 300d2ecda..8b37ee951 100644 --- a/files/firewall/portgroups.local.yaml +++ b/files/firewall/portgroups.local.yaml @@ -1,3 +1,2 @@ firewall: - aliases: - ports: \ No newline at end of file + ports: diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 7f8c01910..a25265c77 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -27,16 +27,16 @@ role: - {{ portgroups.docker_registry }} - {{ portgroups.influxdb }} - {{ portgroups.sensoroni }} - sensor: + sensors: portgroups: - {{ portgroups.beats_5044 }} - {{ portgroups.beats_5644 }} - search_node: + searchnodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.minio }} - {{ portgroups.elasticsearch_node }} - heavy_node: + heavynodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.minio }} @@ -109,17 +109,17 @@ role: {% if ISAIRGAP is sameas true %} - {{ portgroups.yum }} {% endif %} - sensor: + sensors: portgroups: - {{ portgroups.beats_5044 }} - {{ portgroups.beats_5644 }} - search_node: + searchnodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.minio }} - {{ portgroups.elasticsearch_node }} - {{ portgroups.beats_5644 }} - heavy_node: + heavynodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.minio }} @@ -184,16 +184,16 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.sensoroni }} - {{ portgroups.yum }} - sensor: + sensors: portgroups: - {{ portgroups.beats_5044 }} - {{ portgroups.beats_5644 }} - search_node: + searchnodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.minio }} - {{ portgroups.elasticsearch_node }} - heavy_node: + heavynodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.minio }} @@ -260,17 +260,17 @@ role: - {{ portgroups.docker_registry }} - {{ portgroups.influxdb }} - {{ portgroups.sensoroni }} - - {{ portgroups.yum }} - sensor: + - {{ portgroups.yum }} + sensors: portgroups: - {{ portgroups.beats_5044 }} - {{ portgroups.beats_5644 }} - search_node: + searchnodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.minio }} - {{ portgroups.elasticsearch_node }} - heavy_node: + heavynodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.minio }} @@ -339,11 +339,11 @@ role: - {{ portgroups.docker_registry }} - {{ portgroups.influxdb }} - {{ portgroups.sensoroni }} - sensor: + sensors: portgroups: - {{ portgroups.beats_5044 }} - {{ portgroups.beats_5644 }} - search_node: + searchnodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.elasticsearch_node }} @@ -386,7 +386,7 @@ role: portgroups: - {{ portgroups.elasticsearch_rest }} {% if TRUE_CLUSTER %} - search_node: + searchnodes: portgroups: - {{ portgroups.elasticsearch_node }} {% endif %} @@ -472,11 +472,11 @@ role: portgroups: - {{ portgroups.docker_registry }} - {{ portgroups.sensoroni }} - sensor: + sensors: portgroups: - {{ portgroups.beats_5044 }} - {{ portgroups.beats_5644 }} - search_node: + searchnodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.elasticsearch_node }} @@ -511,10 +511,10 @@ role: chain: DOCKER-USER: hostgroups: - sensor: + sensors: portgroups: - {{ portgroups.beats_5644 }} - search_node: + searchnodes: portgroups: - {{ portgroups.redis }} - {{ portgroups.beats_5644 }} diff --git a/salt/firewall/hostgroups/beats_endpoint b/salt/firewall/hostgroups/beats_endpoint new file mode 100644 index 000000000..e69de29bb diff --git a/salt/firewall/hostgroups/beats_endpoint_ssl b/salt/firewall/hostgroups/beats_endpoint_ssl new file mode 100644 index 000000000..e69de29bb diff --git a/salt/firewall/hostgroups/elastic_agent_endpoint b/salt/firewall/hostgroups/elastic_agent_endpoint new file mode 100644 index 000000000..e69de29bb diff --git a/salt/firewall/hostgroups/elasticsearch_rest b/salt/firewall/hostgroups/elasticsearch_rest new file mode 100644 index 000000000..e69de29bb diff --git a/salt/firewall/hostgroups/endgame b/salt/firewall/hostgroups/endgame new file mode 100644 index 000000000..e69de29bb diff --git a/salt/firewall/hostgroups/minion b/salt/firewall/hostgroups/minion new file mode 100644 index 000000000..e69de29bb diff --git a/salt/firewall/hostgroups/strelka_frontend b/salt/firewall/hostgroups/strelka_frontend new file mode 100644 index 000000000..e69de29bb diff --git a/salt/firewall/hostgroups/syslog b/salt/firewall/hostgroups/syslog new file mode 100644 index 000000000..e69de29bb diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index 1d6ba350f..42aad75c6 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -144,4 +144,4 @@ iptables_drop_all_the_things: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/firewall/map.jinja b/salt/firewall/map.jinja index 2c7d03225..45e2989e2 100644 --- a/salt/firewall/map.jinja +++ b/salt/firewall/map.jinja @@ -1,8 +1,8 @@ {% set role = grains.id.split('_') | last %} {% set translated_pillar_assigned_hostgroups = {} %} -{% import_yaml 'firewall/portgroups.yaml' as default_portgroups %} -{% set default_portgroups = default_portgroups.firewall.aliases.ports %} +{% import_yaml 'firewall/ports/ports.yaml' as default_portgroups %} +{% set default_portgroups = default_portgroups.firewall.ports %} {% import_yaml 'firewall/portgroups.local.yaml' as local_portgroups %} {% if local_portgroups.firewall.aliases.ports %} {% set local_portgroups = local_portgroups.firewall.aliases.ports %} @@ -13,7 +13,34 @@ {% set defined_portgroups = portgroups %} {% import_yaml 'firewall/hostgroups.yaml' as default_hostgroups %} -{% import_yaml 'firewall/hostgroups.local.yaml' as local_hostgroups %} +{#% import_yaml 'firewall/hostgroups.local.yaml' as local_hostgroups %#} +{% set local_hostgroups = {'firewall': {'hostgroups': {}}} %} +{% set hostgroup_list = [ + 'analyst', + 'analyst_workstations', + 'eval', + 'heavynodes', + 'idh', + 'manager', + 'minion', + 'receivers', + 'searchnodes', + 'sensors', + 'standalone', + 'beats_endpoint', + 'beats_endpoint_ssl', + 'elasticsearch_rest', + 'elastic_agent_endpoint', + 'endgame', + 'strelka_frontend', + 'syslog' + ] +%} +{% for hg in hostgroup_list %} +{% import_text 'firewall/hostgroups/' ~ hg as hg_ips %} +{% do local_hostgroups.firewall.hostgroups.update({hg: {'ips': {'insert': hg_ips.split(), 'delete': []}}}) %} +{% endfor %} + {% set hostgroups = salt['defaults.merge'](default_hostgroups.firewall.hostgroups, local_hostgroups.firewall.hostgroups, in_place=False) %} {# This block translate the portgroups defined in the pillar to what is defined my portgroups.yaml and portgroups.local.yaml #} diff --git a/salt/firewall/portgroups.yaml b/salt/firewall/portgroups.yaml deleted file mode 100644 index a2780270d..000000000 --- a/salt/firewall/portgroups.yaml +++ /dev/null @@ -1,116 +0,0 @@ -{% if grains.role == 'so-idh' %} - {% from 'idh/opencanary_config.map.jinja' import OPENCANARYCONFIG %} - {% from 'idh/openssh/map.jinja' import openssh_map %} - {% set idh_services = salt['pillar.get']('idh:services', []) %} - {% set ssh_port = openssh_map.config.port %} -{% else %} - {% set ssh_port = 22 %} -{% endif %} - -firewall: - aliases: - ports: - all: - tcp: - - '0:65535' - udp: - - '0:65535' - acng: - tcp: - - 3142 - agrules: - tcp: - - 7788 - beats_5044: - tcp: - - 5044 - beats_5644: - tcp: - - 5644 - beats_5066: - tcp: - - 5066 - cortex: - tcp: - - 9001 - cortex_es_node: - tcp: - - 9500 - cortex_es_rest: - tcp: - - 9400 - docker_registry: - tcp: - - 5000 - elasticsearch_node: - tcp: - - 9300 - elasticsearch_rest: - tcp: - - 9200 - elastic_agent_control: - tcp: - - 8220 - elastic_agent_data: - tcp: - - 5055 - endgame: - tcp: - - 3765 - influxdb: - tcp: - - 8086 - kibana: - tcp: - - 5601 - minio: - tcp: - - 9595 - mysql: - tcp: - - 3306 - nginx: - tcp: - - 80 - - 443 - playbook: - tcp: - - 3200 - redis: - tcp: - - 6379 - - 9696 - salt_manager: - tcp: - - 4505 - - 4506 - sensoroni: - tcp: - - 443 - ssh: - tcp: - - {{ ssh_port }} - strelka_frontend: - tcp: - - 57314 - syslog: - tcp: - - 514 - udp: - - 514 - yum: - tcp: - - 443 - -{% if idh_services is defined %} - {% for service in idh_services %} - {% if service in ["smnp","ntp", "tftp"] %} - {% set proto = 'udp' %} - {% else %} - {% set proto = 'tcp' %} - {% endif %} - idh_{{service}}: - {{proto}}: - - {{ OPENCANARYCONFIG[service~'.port'] }} - {% endfor %} -{% endif %}