From 49b150797d1fa952c1cf8ab0126618007f34f6d5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 12 Jun 2023 16:25:51 -0400 Subject: [PATCH 01/46] 2.4 heavynode changes --- salt/elasticsearch/config.map.jinja | 3 +++ setup/so-setup | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index d367de287..040d4751e 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -22,6 +22,9 @@ {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.roles.extend(['ml', 'master', 'transform']) %} {% endif %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.manager]}}) %} +{% elif grains.id.split('_') | last == 'heavynode' %} + {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['master', 'data_hot', 'remote_cluster_client']}) %} + {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.hostname]}}) %} {% endif %} {% if HIGHLANDER %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.xpack.ml.update({'enabled': true}) %} diff --git a/setup/so-setup b/setup/so-setup index 5f96106a5..53dba8cfd 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -489,9 +489,13 @@ if ! [[ -f $install_opt_file ]]; then check_requirements "heavynode" calculate_useable_cores networking_needful + check_network_manager_conf + set_network_dev_status_list collect_mngr_hostname add_mngr_ip_to_hosts check_manager_connection + detect_cloud + whiptail_sensor_nics set_minion_info whiptail_end_settings From c7c4e65df1e12f5d174a77ea506c70dc8af40e13 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 13 Jun 2023 09:22:17 -0400 Subject: [PATCH 02/46] single-node for heavynode --- salt/elasticsearch/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 4777e3bce..73b0b0282 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -32,7 +32,7 @@ so-elasticsearch: {% endfor %} {% endif %} - environment: - {% if LOGSTASH_NODES | length == 1 %} + {% if LOGSTASH_NODES | length == 1 or GLOBALS.role == 'so-heavynode' %} - discovery.type=single-node {% endif %} - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true From 2c4eccd7e084aaae86aa2508ef02c142d4009a46 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Jun 2023 10:40:05 -0400 Subject: [PATCH 03/46] 2.4 heavynode changes --- salt/elasticsearch/config.map.jinja | 3 +-- salt/logstash/defaults.yaml | 1 + salt/logstash/enabled.sls | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 040d4751e..832616e38 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -23,8 +23,7 @@ {% endif %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.manager]}}) %} {% elif grains.id.split('_') | last == 'heavynode' %} - {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['master', 'data_hot', 'remote_cluster_client']}) %} - {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.hostname]}}) %} + {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} {% endif %} {% if HIGHLANDER %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.xpack.ml.update({'enabled': true}) %} diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index a273476e1..8d27730b2 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -8,6 +8,7 @@ logstash: receiver: - receiver heavynode: + - manager - search searchnode: - search diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index 91433cba8..bb1c2f472 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -58,7 +58,7 @@ so-logstash: - /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro {% endif %} - {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-eval','so-fleet'] %} + {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-eval','so-fleet', 'so-heavynode'] %} - /opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.crt:/usr/share/logstash/elasticfleet-logstash.crt:ro - /opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.p8:/usr/share/logstash/elasticfleet-logstash.key:ro {% endif %} From 934b0f45a137e422359fc307a8eae260e8299ab5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Jun 2023 15:08:07 -0400 Subject: [PATCH 04/46] allow receiver to connect to salt manager --- salt/firewall/defaults.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 176d12ec2..bccfe5c80 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -486,6 +486,9 @@ firewall: heavynode: portgroups: - salt_manager + receiver: + portgroups: + - salt_manager customhostgroup0: portgroups: [] customhostgroup1: @@ -638,6 +641,9 @@ firewall: heavynode: portgroups: - salt_manager + receiver: + portgroups: + - salt_manager customhostgroup0: portgroups: [] customhostgroup1: @@ -806,6 +812,9 @@ firewall: heavynode: portgroups: - salt_manager + receiver: + portgroups: + - salt_manager customhostgroup0: portgroups: [] customhostgroup1: From 2db95fe1b41313c0de462586043cb0846881a981 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 14 Jun 2023 15:24:14 -0400 Subject: [PATCH 05/46] fw rules for receiver to managers --- salt/firewall/defaults.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index bccfe5c80..450f6e194 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -417,6 +417,11 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + receiver: + portgroups: + - yum + - docker_registry + - influxdb self: portgroups: - syslog @@ -572,6 +577,11 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + receiver: + portgroups: + - yum + - docker_registry + - influxdb self: portgroups: - syslog @@ -737,6 +747,11 @@ firewall: - redis - elasticsearch_rest - elasticsearch_node + receiver: + portgroups: + - yum + - docker_registry + - influxdb self: portgroups: - syslog From 469390696e551d1bdeed0c62f445d6178ba6dee5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 15 Jun 2023 11:04:16 -0400 Subject: [PATCH 06/46] 2.4 receiver changes --- salt/firewall/defaults.yaml | 9 +++++++++ salt/logstash/enabled.sls | 2 +- salt/ssl/init.sls | 2 +- salt/vars/receiver.map.jinja | 1 + 4 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 salt/vars/receiver.map.jinja diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 450f6e194..638bc85ff 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -422,6 +422,9 @@ firewall: - yum - docker_registry - influxdb + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update self: portgroups: - syslog @@ -582,6 +585,9 @@ firewall: - yum - docker_registry - influxdb + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update self: portgroups: - syslog @@ -752,6 +758,9 @@ firewall: - yum - docker_registry - influxdb + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update self: portgroups: - syslog diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index bb1c2f472..abc28cfe6 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -58,7 +58,7 @@ so-logstash: - /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro {% endif %} - {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-eval','so-fleet', 'so-heavynode'] %} + {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-eval','so-fleet', 'so-heavynode', 'so-receiver'] %} - /opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.crt:/usr/share/logstash/elasticfleet-logstash.crt:ro - /opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.p8:/usr/share/logstash/elasticfleet-logstash.key:ro {% endif %} diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index e077d55d0..96953ffff 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -140,7 +140,7 @@ rediskeyperms: - group: 939 {% endif %} -{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet'] %} +{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet', 'so-receiver'] %} # Create cert for Elastic Fleet Host etc_elasticfleet_key: diff --git a/salt/vars/receiver.map.jinja b/salt/vars/receiver.map.jinja new file mode 100644 index 000000000..964f69663 --- /dev/null +++ b/salt/vars/receiver.map.jinja @@ -0,0 +1 @@ +{% set ROLE_GLOBALS = {} %} From 5de9e5baf49e5cef7e6e13a1c8e480fd1e4e9923 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 15 Jun 2023 14:46:46 -0400 Subject: [PATCH 07/46] allow sensor to logstash on receiver --- salt/firewall/defaults.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 638bc85ff..45567de91 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -1161,7 +1161,9 @@ firewall: hostgroups: sensor: portgroups: + - beats_5044 - beats_5644 + - elastic_agent_data searchnode: portgroups: - redis From 36272efda7412b731728bbd5904e995bc33ee399 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 22 Jun 2023 09:46:42 -0400 Subject: [PATCH 08/46] create ES_LOGSTASH_NODES which removes heavynodes --- salt/elasticsearch/config.map.jinja | 18 ++++++++++++++---- salt/elasticsearch/enabled.sls | 3 ++- salt/logstash/map.jinja | 1 + 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 832616e38..1b8728bf5 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,13 +1,23 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %} -{% from 'logstash/map.jinja' import LOGSTASH_NODES with context %} {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} +{# ES_LOGSTASH_NODES is the same as LOGSTASH_NODES from logstash/map.jinja but heavynodes are removed #} +{% set ES_LOGSTASH_NODES = [] %} +{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} +{% for node_type, node_details in node_data.items() | sort %} +{% if node_type != 'heavynode' %} +{% for hostname in node_data[node_type].keys() %} +{% do ES_LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %} +{% endfor %} +{% endif %} +{% endfor %} + {% if grains.id.split('_') | last in ['manager','managersearch','standalone'] %} - {% if LOGSTASH_NODES | length > 1 %} + {% if ES_LOGSTASH_NODES | length > 1 %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %} - {% for NODE in LOGSTASH_NODES %} + {% for NODE in ES_LOGSTASH_NODES %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %} {% endfor %} {% if grains.id.split('_') | last == 'manager' %} @@ -23,7 +33,7 @@ {% endif %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.manager]}}) %} {% elif grains.id.split('_') | last == 'heavynode' %} - {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} + {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client', 'ingest']}) %} {% endif %} {% if HIGHLANDER %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.xpack.ml.update({'enabled': true}) %} diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 73b0b0282..3876c5853 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -8,6 +8,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} {% from 'logstash/map.jinja' import LOGSTASH_NODES %} +{% from 'elasticsearch/config.map.jinja' import ES_LOGSTASH_NODES %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} {% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} @@ -32,7 +33,7 @@ so-elasticsearch: {% endfor %} {% endif %} - environment: - {% if LOGSTASH_NODES | length == 1 or GLOBALS.role == 'so-heavynode' %} + {% if ES_LOGSTASH_NODES | length == 1 or GLOBALS.role == 'so-heavynode' %} - discovery.type=single-node {% endif %} - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index 69e102e78..4098df21f 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -8,6 +8,7 @@ {% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %} {% set REDIS_NODES = [] %} +{# LOGSTASH_NODES is the same as ES_LOGSTASH_NODES from elasticsearch/config.map.jinja but heavynodes are present #} {% set LOGSTASH_NODES = [] %} {% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} From 6c4c815683d6c2086196e15d3648112b13eb1a33 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 22 Jun 2023 13:04:20 -0400 Subject: [PATCH 09/46] change so-elasticsearch-cluster settings to include heavynode, and only run on managers --- salt/elasticsearch/enabled.sls | 2 ++ .../so-elasticsearch-cluster-settings | 21 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 3876c5853..e28ca5fdf 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -145,6 +145,7 @@ es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: {% endfor %} {% endif %} +{% if GLOBALS.role in GLOBALS.manager_roles %} so-es-cluster-settings: cmd.run: - name: /usr/sbin/so-elasticsearch-cluster-settings @@ -153,6 +154,7 @@ so-es-cluster-settings: - require: - docker_container: so-elasticsearch - file: elasticsearch_sbin_jinja +{% endif %} so-elasticsearch-ilm-policy-load: cmd.run: diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings index 83a997845..c3befcb91 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings @@ -3,6 +3,8 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +{%- from 'vars/globals.map.jinja' import GLOBALS %} +{%- set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} . /usr/sbin/so-common @@ -14,17 +16,17 @@ COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 30 ]]; do curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://localhost:"$ELASTICSEARCH_PORT" - if [ $? -eq 0 ]; then + if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" - break + break else ((COUNT+=1)) - sleep 1 + sleep 1 echo -n "." fi done -if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then +if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then echo echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'docker ps' \n -running 'sudo so-elastic-restart'" echo @@ -32,9 +34,12 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then exit fi -# Check to see if config already exists -CLUSTER_SETTINGS=$(so-elasticsearch-query _cluster/settings | jq .persistent.cluster.remote) -if [[ ! -z "$CLUSTER_SETTINGS" ]]; then +{%- if GLOBALS.role in [ 'so-manager', 'so-managersearch', 'so-standalone'] %} echo "Applying cross cluster search config..." so-elasticsearch-query _cluster/settings -d "{\"persistent\": {\"cluster\": {\"remote\": {\"{{ GLOBALS.manager }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" -XPUT -fi +{%- if node_data['heavynode'] is defined %} +{%- for hostname, node_details in node_data['heavynode'].items() %} + so-elasticsearch-query _cluster/settings -d "{\"persistent\": {\"cluster\": {\"remote\": {\"{{ hostname }}\": {\"seeds\": [\"{{node_details.ip}}:9300\"]}}}}}" -XPUT +{%- endfor %} +{%- endif %} +{%- endif %} From 19469205e1fda3aaf6cbc74ec862b805aafdf03e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 22 Jun 2023 13:12:47 -0400 Subject: [PATCH 10/46] include eval and import in so-elasticsearch-cluster-settings --- .../tools/sbin_jinja/so-elasticsearch-cluster-settings | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings index c3befcb91..9048b85b1 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings @@ -34,7 +34,7 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then exit fi -{%- if GLOBALS.role in [ 'so-manager', 'so-managersearch', 'so-standalone'] %} +{%- if GLOBALS.role in GLOBALS.manager_roles %} echo "Applying cross cluster search config..." so-elasticsearch-query _cluster/settings -d "{\"persistent\": {\"cluster\": {\"remote\": {\"{{ GLOBALS.manager }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" -XPUT {%- if node_data['heavynode'] is defined %} From d0d7ab57cabe31a13efbb5b439472e907f6ff392 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 22 Jun 2023 16:02:17 -0400 Subject: [PATCH 11/46] Add Elastic Agent container for Heavy Nodes --- salt/allowed_states.map.jinja | 1 + salt/elasticagent/config.sls | 47 +++++++ salt/elasticagent/defaults.yaml | 2 + salt/elasticagent/disabled.sls | 27 ++++ salt/elasticagent/enabled.sls | 65 ++++++++++ .../files/elastic-agent.yml.jinja | 118 ++++++++++++++++++ salt/elasticagent/init.sls | 13 ++ salt/elasticagent/map.jinja | 7 ++ salt/elasticagent/sostatus.sls | 21 ++++ salt/elasticfleet/config.sls | 6 +- salt/top.sls | 1 + 11 files changed, 305 insertions(+), 3 deletions(-) create mode 100644 salt/elasticagent/config.sls create mode 100644 salt/elasticagent/defaults.yaml create mode 100644 salt/elasticagent/disabled.sls create mode 100644 salt/elasticagent/enabled.sls create mode 100644 salt/elasticagent/files/elastic-agent.yml.jinja create mode 100644 salt/elasticagent/init.sls create mode 100644 salt/elasticagent/map.jinja create mode 100644 salt/elasticagent/sostatus.sls diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index e7a9a0491..206c2fad6 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -46,6 +46,7 @@ 'pcap', 'suricata', 'healthcheck', + 'elasticagent', 'schedule', 'tcpreplay', 'docker_clean' diff --git a/salt/elasticagent/config.sls b/salt/elasticagent/config.sls new file mode 100644 index 000000000..18d0e482e --- /dev/null +++ b/salt/elasticagent/config.sls @@ -0,0 +1,47 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% if sls.split('.')[0] in allowed_states %} + +# Add EA Group +elasticagentgroup: + group.present: + - name: elastic-agent + - gid: 949 + +# Add EA user +elastic-agent: + user.present: + - uid: 949 + - gid: 949 + - home: /opt/so/conf/elastic-agent + - createhome: False + +elasticagentconfdir: + file.directory: + - name: /opt/so/conf/elastic-agent + - user: 949 + - group: 939 + - makedirs: True + +# Create config +create-elastic-agent-config: + file.managed: + - name: /opt/so/conf/elastic-agent/elastic-agent.yml + - source: salt://elasticagent/files/elastic-agent.yml.jinja + - user: 949 + - group: 939 + - template: jinja + + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticagent/defaults.yaml b/salt/elasticagent/defaults.yaml new file mode 100644 index 000000000..f9b1bd67a --- /dev/null +++ b/salt/elasticagent/defaults.yaml @@ -0,0 +1,2 @@ +elasticagent: + enabled: False \ No newline at end of file diff --git a/salt/elasticagent/disabled.sls b/salt/elasticagent/disabled.sls new file mode 100644 index 000000000..a6d72b193 --- /dev/null +++ b/salt/elasticagent/disabled.sls @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} + +include: + - elasticagent.sostatus + +so-elastic-agent: + docker_container.absent: + - force: True + +so-elastic-agent_so-status.disabled: + file.comment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-elastic-agent$ + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticagent/enabled.sls b/salt/elasticagent/enabled.sls new file mode 100644 index 000000000..52467a54e --- /dev/null +++ b/salt/elasticagent/enabled.sls @@ -0,0 +1,65 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'docker/docker.map.jinja' import DOCKER %} + + +include: + - elasticagent.config + - elasticagent.sostatus + +so-elastic-agent: + docker_container.running: + - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }} + - name: so-elastic-agent + - hostname: {{ GLOBALS.hostname }} + - detach: True + - user: 949 + - networks: + - sobridge: + - ipv4_address: {{ DOCKER.containers['so-elastic-agent'].ip }} + - extra_hosts: + - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} + - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }} + {% if DOCKER.containers['so-elastic-agent'].extra_hosts %} + {% for XTRAHOST in DOCKER.containers['so-elastic-agent'].extra_hosts %} + - {{ XTRAHOST }} + {% endfor %} + {% endif %} + - port_bindings: + {% for BINDING in DOCKER.containers['so-elastic-agent'].port_bindings %} + - {{ BINDING }} + {% endfor %} + - binds: + - /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro + {% if DOCKER.containers['so-elastic-agent'].custom_bind_mounts %} + {% for BIND in DOCKER.containers['so-elastic-agent'].custom_bind_mounts %} + - {{ BIND }} + {% endfor %} + {% endif %} + - environment: + {% if DOCKER.containers['so-elastic-agent'].extra_env %} + {% for XTRAENV in DOCKER.containers['so-elastic-agent'].extra_env %} + - {{ XTRAENV }} + {% endfor %} + {% endif %} + + +delete_so-elastic-agent_so-status.disabled: + file.uncomment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-elastic-agent$ + + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticagent/files/elastic-agent.yml.jinja b/salt/elasticagent/files/elastic-agent.yml.jinja new file mode 100644 index 000000000..3eb63962b --- /dev/null +++ b/salt/elasticagent/files/elastic-agent.yml.jinja @@ -0,0 +1,118 @@ +{% from 'vars/globals.map.jinja' import GLOBALS %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} + +id: aea1ba80-1065-11ee-a369-97538913b6a9 +revision: 2 +outputs: + default: + type: elasticsearch + hosts: + - 'https://{{ GLOBALS.hostname }}:9200' + username: '{{ ES_USER }}' + password: '{{ ES_PASS }}' +output_permissions: {} +agent: + download: + sourceURI: 'http://{{ GLOBALS.manager }}:8443/artifacts/' + monitoring: + enabled: false + logs: false + metrics: false + features: {} +inputs: + - id: logfile-logs-80ffa884-2cfc-459a-964a-34df25714d85 + name: suricata-logs + revision: 1 + type: logfile + use_output: default + meta: + package: + name: log + version: 1.1.2 + data_stream: + namespace: so + package_policy_id: 80ffa884-2cfc-459a-964a-34df25714d85 + streams: + - id: logfile-log.log-80ffa884-2cfc-459a-964a-34df25714d85 + data_stream: + dataset: suricata + paths: + - /nsm/suricata/eve*.json + processors: + - add_fields: + target: event + fields: + category: network + module: suricata + pipeline: suricata.common + - id: logfile-logs-90103ac4-f6bd-4a4a-b596-952c332390fc + name: strelka-logs + revision: 1 + type: logfile + use_output: default + meta: + package: + name: log + version: 1.1.2 + data_stream: + namespace: so + package_policy_id: 90103ac4-f6bd-4a4a-b596-952c332390fc + streams: + - id: logfile-log.log-90103ac4-f6bd-4a4a-b596-952c332390fc + data_stream: + dataset: strelka + paths: + - /nsm/strelka/log/strelka.log + processors: + - add_fields: + target: event + fields: + category: file + module: strelka + pipeline: strelka.file + - id: logfile-logs-6197fe84-9b58-4d9b-8464-3d517f28808d + name: zeek-logs + revision: 1 + type: logfile + use_output: default + meta: + package: + name: log + version: 1.1.2 + data_stream: + namespace: so + package_policy_id: 6197fe84-9b58-4d9b-8464-3d517f28808d + streams: + - id: logfile-log.log-6197fe84-9b58-4d9b-8464-3d517f28808d + data_stream: + dataset: zeek + paths: + - /nsm/zeek/logs/current/*.log + processors: + - dissect: + tokenizer: '/nsm/zeek/logs/current/%{pipeline}.log' + field: log.file.path + trim_chars: .log + target_prefix: '' + - script: + lang: javascript + source: | + function process(event) { + var pl = event.Get("pipeline"); + event.Put("@metadata.pipeline", "zeek." + pl); + } + - add_fields: + target: event + fields: + category: network + module: zeek + - add_tags: + tags: ics + when: + regexp: + pipeline: >- + ^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm* + exclude_files: + - >- + broker|capture_loss|cluster|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout.log$ diff --git a/salt/elasticagent/init.sls b/salt/elasticagent/init.sls new file mode 100644 index 000000000..3ea474026 --- /dev/null +++ b/salt/elasticagent/init.sls @@ -0,0 +1,13 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'elasticagent/map.jinja' import ELASTICAGENTMERGED %} + +include: +{% if ELASTICAGENTMERGED.enabled %} + - elasticagent.enabled +{% else %} + - elasticagent.disabled +{% endif %} diff --git a/salt/elasticagent/map.jinja b/salt/elasticagent/map.jinja new file mode 100644 index 000000000..f48172502 --- /dev/null +++ b/salt/elasticagent/map.jinja @@ -0,0 +1,7 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + +{% import_yaml 'elasticagent/defaults.yaml' as ELASTICAGENTDEFAULTS %} +{% set ELASTICAGENTMERGED = salt['pillar.get']('elasticagent', ELASTICAGENTDEFAULTS.elasticagent, merge=True) %} diff --git a/salt/elasticagent/sostatus.sls b/salt/elasticagent/sostatus.sls new file mode 100644 index 000000000..e2cbc9b8e --- /dev/null +++ b/salt/elasticagent/sostatus.sls @@ -0,0 +1,21 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} + +append_so-elastic-agent_so-status.conf: + file.append: + - name: /opt/so/conf/so-status/so-status.conf + - text: so-elastic-agent + - unless: grep -q so-elastic-agent$ /opt/so/conf/so-status/so-status.conf + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index cc3b0675f..9bfb6e34d 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -8,13 +8,13 @@ {% if sls.split('.')[0] in allowed_states %} # Add EA Group -elasticsagentgroup: +elasticfleetgroup: group.present: - - name: elastic-agent + - name: elastic-fleet - gid: 947 # Add EA user -elastic-agent: +elastic-fleet: user.present: - uid: 947 - gid: 947 diff --git a/salt/top.sls b/salt/top.sls index 229557575..11a594f29 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -209,6 +209,7 @@ base: - suricata - zeek - elasticfleet.install_agent_grid + - elasticagent - docker_clean '*_import and G@saltversion:{{saltversion}}': From 8de01625a8a5841eca1658df1bca09ebefdfe72c Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 22 Jun 2023 16:02:42 -0400 Subject: [PATCH 12/46] Add Elastic Agent container for Heavy Nodes --- pillar/top.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pillar/top.sls b/pillar/top.sls index 75117e35f..464307065 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -189,6 +189,8 @@ base: - logstash.adv_logstash - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch + - elasticagent.soc_elasticagent + - elasticagent.adv_elasticagent - curator.soc_curator - curator.adv_curator - redis.soc_redis From 662db4185738c8827d690fa41878742223012701 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 23 Jun 2023 13:17:38 +0000 Subject: [PATCH 13/46] Add default packages --- salt/elasticfleet/defaults.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 8c858c711..83261dc17 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -23,3 +23,11 @@ elasticfleet: - stats - stderr - stdout + packages: + - aws + - azure + - cloudflare + - fim + - github + - google_workspace + - 1password From 94fe456e28bea732c7c0ffbb607ab6fa4cf9bafd Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 23 Jun 2023 13:19:20 +0000 Subject: [PATCH 14/46] Add package functions --- .../tools/sbin/so-elastic-fleet-common | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common index e56ee7f0a..73c36e5c8 100755 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common @@ -51,6 +51,21 @@ elastic_fleet_integration_update() { curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" } +elastic_fleet_package_version_check() { + PACKAGE=$1 + curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.version' +} + +elastic_fleet_package_install() { + PKGKEY=$1 + curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PKGKEY" +} + +elastic_fleet_package_is_installed() { + PACKAGE=$1 + curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.status' +} + elastic_fleet_policy_create() { NAME=$1 From a3c7e40c40bc5a95979009bbaf9e5295038503f5 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 23 Jun 2023 13:20:05 +0000 Subject: [PATCH 15/46] Add package load command --- .../sbin_jinja/so-elastic-fleet-package-load | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load new file mode 100644 index 000000000..c1e14f64f --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. +{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} +{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} + +. /usr/sbin/so-elastic-fleet-common + +{%- for PACKAGE in SUPPORTED_PACKAGES %} +echo "Setting up {{ PACKAGE }} package..." +VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") +elastic_fleet_package_install "{{ PACKAGE }}-$VERSION" +echo +{%- endfor %} +echo From 2c42d4b19ef990c1824e08192ec1f45457381f53 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 23 Jun 2023 13:22:51 +0000 Subject: [PATCH 16/46] Add package check to so-elasticsearch-templates-load --- .../sbin/so-elasticsearch-templates-load | 63 ------------------- 1 file changed, 63 deletions(-) delete mode 100755 salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load deleted file mode 100755 index bce8af1ff..000000000 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -. /usr/sbin/so-common - -default_conf_dir=/opt/so/conf - -# Define a default directory to load pipelines from -ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/" - -# Wait for ElasticSearch to initialize -echo -n "Waiting for ElasticSearch..." -COUNT=0 -ELASTICSEARCH_CONNECTED="no" -while [[ "$COUNT" -le 240 ]]; do - so-elasticsearch-query / -k --output /dev/null --silent --head --fail - if [ $? -eq 0 ]; then - ELASTICSEARCH_CONNECTED="yes" - echo "connected!" - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." - fi -done -if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then - echo - echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" - echo - exit 1 -fi - -set -e - -cd ${ELASTICSEARCH_TEMPLATES}/component/ecs - -echo "Loading ECS component templates..." -for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE-mappings"; so-elasticsearch-query _component_template/$TEMPLATE-mappings -d@$i -XPUT 2>/dev/null; echo; done - -cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent - -echo "Loading Elastic Agent component templates..." -for i in *; do TEMPLATE=${i::-5}; echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done - -# Load SO-specific component templates -cd ${ELASTICSEARCH_TEMPLATES}/component/so - -echo "Loading Security Onion component templates..." -for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done -echo - -# Load SO index templates -cd ${ELASTICSEARCH_TEMPLATES}/index - -echo "Loading Security Onion index templates..." -for i in *; do TEMPLATE=${i::-14}; echo "$TEMPLATE"; so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done -echo - -cd - >/dev/null From d8700137d27daf81a707c21c2493a88f2cc50258 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 23 Jun 2023 13:23:29 +0000 Subject: [PATCH 17/46] Add updated so-elasticsearch-templates-load --- .../so-elasticsearch-templates-load | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100755 salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load new file mode 100755 index 000000000..60e32c12b --- /dev/null +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -0,0 +1,76 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. +{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} +{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} + +. /usr/sbin/so-common +. /usr/sbin/so-elastic-fleet-common + + +default_conf_dir=/opt/so/conf + +# Define a default directory to load pipelines from +ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/" + +# Wait for ElasticSearch to initialize +echo -n "Waiting for ElasticSearch..." +COUNT=0 +ELASTICSEARCH_CONNECTED="no" +while [[ "$COUNT" -le 240 ]]; do + so-elasticsearch-query / -k --output /dev/null --silent --head --fail + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi +done +if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo + exit 1 +fi + +SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') +INSTALLED=$(elastic_fleet_package_is_installed {{ SUPPORTED_PACKAGES[0] }} ) +if [ "$INSTALLED" != "installed" ]; then + echo + echo "Packages not yet installed." + echo + exit 0 +fi + +set -e + +cd ${ELASTICSEARCH_TEMPLATES}/component/ecs + +echo "Loading ECS component templates..." +for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE-mappings"; so-elasticsearch-query _component_template/$TEMPLATE-mappings -d@$i -XPUT 2>/dev/null; echo; done + +cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent + +echo "Loading Elastic Agent component templates..." +for i in *; do TEMPLATE=${i::-5}; echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done + +# Load SO-specific component templates +cd ${ELASTICSEARCH_TEMPLATES}/component/so + +echo "Loading Security Onion component templates..." +for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done +echo + +# Load SO index templates +cd ${ELASTICSEARCH_TEMPLATES}/index + +echo "Loading Security Onion index templates..." +for i in *; do TEMPLATE=${i::-14}; echo "$TEMPLATE"; so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done +echo + +cd - >/dev/null From e2cca917c195710c9e2416d4c0fda012143aa77a Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 23 Jun 2023 13:26:06 +0000 Subject: [PATCH 18/46] Add package load command to Fleet setup --- .../elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load | 0 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup | 2 ++ 2 files changed, 2 insertions(+) mode change 100644 => 100755 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load old mode 100644 new mode 100755 diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 94a42a70a..e7b8e115d 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -48,6 +48,8 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl printf "\n\n" ### Create Policies & Associated Integration Configuration ### +# Load packages +/usr/sbin/so-elastic-fleet-package-load # Manager Fleet Server Host elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120" From 5e8748c436d56bc7307ee68cf3e490c4bccb3cb3 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 23 Jun 2023 13:28:01 +0000 Subject: [PATCH 19/46] Load Elasticsearch templates --- salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index e7b8e115d..c63181a79 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -51,6 +51,9 @@ printf "\n\n" # Load packages /usr/sbin/so-elastic-fleet-package-load +# Load Elasticsearch templates +/usr/sbin/so-elasticsearch-templates-load + # Manager Fleet Server Host elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120" From e995576b1d91dc0fbd9e60a9610d50be60a21078 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 23 Jun 2023 09:41:49 -0400 Subject: [PATCH 20/46] Remove extra templates --- salt/elasticsearch/defaults.yaml | 3235 ------------------------------ 1 file changed, 3235 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index d27f291eb..e6fec1c6b 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -873,440 +873,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-aws: - warm: 7 - close: 30 - delete: 365 - index_sorting: False - index_template: - index_patterns: - - so-aws-* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - aws-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-azure: - warm: 7 - close: 30 - delete: 365 - index_sorting: False - index_template: - index_patterns: - - so-azure-* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - azure-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-barracuda: - warm: 7 - close: 30 - delete: 365 - index_sorting: False - index_template: - index_patterns: - - so-barracuda-* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-beats: - warm: 7 - close: 30 - delete: 365 - index_sorting: False - index_template: - index_patterns: - - so-beats-* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - - winlog-mappings - - dtc-winlog-mappings - priority: 500 - so-bluecoat: - warm: 7 - close: 30 - delete: 365 - index_sorting: False - index_template: - index_patterns: - - so-bluecoat-* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 so-case: index_sorting: False index_template: @@ -1336,258 +902,6 @@ elasticsearch: - case-mappings - case-settings priority: 500 - so-cef: - index_sorting: False - index_template: - index_patterns: - - so-cef* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - cef-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-checkpoint: - index_sorting: False - index_template: - index_patterns: - - so-checkpoint* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - checkpoint-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-cisco: - index_sorting: False - index_template: - index_patterns: - - so-cisco* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - cisco-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 so-common: warm: 7 close: 30 @@ -1678,278 +992,6 @@ elasticsearch: - common-dynamic-mappings - winlog-mappings priority: 1 - so-cyberark: - index_sorting: False - index_template: - index_patterns: - - so-cyberark* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - cyberark-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-cylance: - index_sorting: False - index_template: - index_patterns: - - so-cylance* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-elasticsearch: - index_sorting: False - index_template: - index_patterns: - - logs-elasticsearch-default* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - lifecycle: - name: so-elasticsearch-logs - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - elasticsearch-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} so-endgame: index_sorting: False index_template: @@ -2035,509 +1077,6 @@ elasticsearch: - common-dynamic-mappings - winlog-mappings priority: 500 - so-f5: - index_sorting: False - index_template: - index_patterns: - - so-f5* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-firewall: - index_sorting: False - index_template: - index_patterns: - - so-firewall* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - syslog-mappings - - dtc-syslog-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-flow: - index_sorting: False - index_template: - index_patterns: - - so-flow* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-fortinet: - index_sorting: False - index_template: - index_patterns: - - so-fortinet* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - fortinet-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-gcp: - index_sorting: False - index_template: - index_patterns: - - so-gcp* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - gcp-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-google_workspace: - index_sorting: False - index_template: - index_patterns: - - so-google_workspace* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - google_workspace-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 so-idh: warm: 7 close: 30 @@ -2724,89 +1263,6 @@ elasticsearch: min_age: 365d actions: delete: {} - so-imperva: - index_sorting: False - index_template: - index_patterns: - - so-imperva* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 so-import: index_sorting: False index_template: @@ -2913,277 +1369,6 @@ elasticsearch: min_age: 365d actions: delete: {} - so-infoblox: - index_sorting: False - index_template: - index_patterns: - - so-infoblox* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - - winlog-mappings - priority: 500 - so-juniper: - index_sorting: False - index_template: - index_patterns: - - so-juniper* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - juniper-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-kibana: - index_sorting: False - index_template: - index_patterns: - - so-kibana* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - kibana-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} so-kratos: warm: 7 close: 30 @@ -3391,843 +1576,6 @@ elasticsearch: min_age: 365d actions: delete: {} - so-microsoft: - index_sorting: False - index_template: - index_patterns: - - so-microsoft* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - microsoft-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-misp: - index_sorting: False - index_template: - index_patterns: - - so-misp* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - misp-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-netflow: - index_sorting: False - index_template: - index_patterns: - - so-netflow* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - netflow-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-netscout: - index_sorting: False - index_template: - index_patterns: - - so-netscout* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-o365: - index_sorting: False - index_template: - index_patterns: - - so-o365* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - o365-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-okta: - index_sorting: False - index_template: - index_patterns: - - so-okta* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - okta-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-osquery: - index_sorting: False - index_template: - index_patterns: - - so-osquery* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - - winlog-mappings - priority: 500 - so-ossec: - index_sorting: False - index_template: - index_patterns: - - so-ossec* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - - winlog-mappings - priority: 500 - so-proofpoint: - index_sorting: False - index_template: - index_patterns: - - so-proofpoint* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-radware: - index_sorting: False - index_template: - index_patterns: - - so-radware* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 so-redis: index_sorting: False index_template: @@ -4333,423 +1681,6 @@ elasticsearch: min_age: 365d actions: delete: {} - so-snort: - index_sorting: False - index_template: - index_patterns: - - so-snort* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-snyk: - index_sorting: False - index_template: - index_patterns: - - so-snyk* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - snyk-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-sonicwall: - index_sorting: False - index_template: - index_patterns: - - so-sonicwall* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-sophos: - index_sorting: False - index_template: - index_patterns: - - so-sophos* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - sophos-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-squid: - index_sorting: False - index_template: - index_patterns: - - so-squid* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 so-strelka: index_sorting: False index_template: @@ -4959,89 +1890,6 @@ elasticsearch: min_age: 365d actions: delete: {} - so-tomcat: - index_sorting: False - index_template: - index_patterns: - - so-tomcat* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 so-zeek: index_sorting: False index_template: @@ -5150,86 +1998,3 @@ elasticsearch: min_age: 365d actions: delete: {} - so-zscaler: - index_sorting: False - index_template: - index_patterns: - - so-zscaler* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 From 62fa15c63ed86a5e5bee08d923b67ac801e4ddf3 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 23 Jun 2023 14:43:15 +0000 Subject: [PATCH 21/46] Add more templates --- salt/elasticsearch/defaults.yaml | 792 +++++++++++++++++++++++++++++++ 1 file changed, 792 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index e6fec1c6b..5442969e5 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -278,6 +278,798 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false + so-logs-aws.cloudtrail: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.cloudtrail-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-aws.cloudtrail@package" + - "logs-aws.cloudtrail@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-aws.cloudwatch_logs: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.cloudwatch_logs-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-aws.cloudwatch_logs@package" + - "logs-aws.cloudwatch_logs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-aws.ec2_logs: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.ec2_logs-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-aws.ec2_logs@package" + - "logs-aws.ec2_logs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-aws.elb_logs: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.elb_logs-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-aws.elb_logs@package" + - "logs-aws.elb_logs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-aws.firewall_logs: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.firewall_logs-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-aws.firewall_logs@package" + - "logs-aws.firewall_logs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-aws.route53_public_logs: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.route53_public_logs-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-aws.route53_public_logs@package" + - "logs-aws.route53_public_logs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-aws.route53.resolver_logs: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.route53.resolver_logs-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-aws.route53.resolver_logs@package" + - "logs-aws.route53.resolver_logs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-aws.s3access: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.s3access-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-aws.s3access@package" + - "logs-aws.s3access@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-aws.vpcflow: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.vpcflow-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-aws.vpcflow@package" + - "logs-aws.vpcflow@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-aws.waf: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.waf-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-aws.waf@package" + - "logs-aws.waf@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-azure.activitylogs: + index_sorting: False + index_template: + index_patterns: + - "logs-azure.activitylogs-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-azure.activitylogs@package" + - "logs-azure.activitylogs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-azure.application_gateway: + index_sorting: False + index_template: + index_patterns: + - "logs-azure.application_gateway-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-azure.application_gateway@package" + - "logs-azure.application_gateway@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-azure.auditlogs: + index_sorting: False + index_template: + index_patterns: + - "logs-azure.auditlogs-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-azure.auditlogs@package" + - "logs-azure.auditlogs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-azure.eventhub: + index_sorting: False + index_template: + index_patterns: + - "logs-azure.eventhub-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-azure.eventhub@package" + - "logs-azure.eventhub@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-azure.firewall_logs: + index_sorting: False + index_template: + index_patterns: + - "logs-azure.firewall_logs-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-azure.firewall_logs@package" + - "logs-azure.firewall_logs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-azure.identity_protect: + index_sorting: False + index_template: + index_patterns: + - "logs-azure.identity_protect-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-azure.identity_protect@package" + - "logs-azure.identity_protect@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-azure.platformlogs: + index_sorting: False + index_template: + index_patterns: + - "logs-azure.platformlogs-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-azure.platformlogs@package" + - "logs-azure.platformlogs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-azure.provisioning: + index_sorting: False + index_template: + index_patterns: + - "logs-azure.provisioning-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-azure.provisioning@package" + - "logs-azure.provisioning@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-azure.signinlogs: + index_sorting: False + index_template: + index_patterns: + - "logs-azure.signinlogs-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-azure.signinlogs@package" + - "logs-azure.signinlogs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-azure.springcloudlogs: + index_sorting: False + index_template: + index_patterns: + - "logs-azure.springcloudlogs-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-azure.springcloudlogs@package" + - "logs-azure.springcloudlogs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-cloudflare-audit: + index_sorting: False + index_template: + index_patterns: + - "logs-cloudflare-audit-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-cloudflare-audit@package" + - "logs-cloudflare-audit@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-cloudflare-logpull: + index_sorting: False + index_template: + index_patterns: + - "logs-cloudflare-logpull-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-cloudflare-logpull@package" + - "logs-cloudflare-logpull@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-fim.event: + index_sorting: False + index_template: + index_patterns: + - "logs-fim.event-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-fim.event@package" + - "logs-fim.event@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-github.audit: + index_sorting: False + index_template: + index_patterns: + - "logs-github.audit-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-github.audit@package" + - "logs-github.audit@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-github.code_scanning: + index_sorting: False + index_template: + index_patterns: + - "logs-github.code_scanning-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-github.code_scanning@package" + - "logs-github.code_scanning@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-github.dependabot: + index_sorting: False + index_template: + index_patterns: + - "logs-github.dependabot-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-github.dependabot@package" + - "logs-github.dependabot@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-github.issues: + index_sorting: False + index_template: + index_patterns: + - "logs-github.issues-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-github.issues@package" + - "logs-github.issues@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-github.secret_scanning: + index_sorting: False + index_template: + index_patterns: + - "logs-github.secret_scanning-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-github.secret_scanning@package" + - "logs-github.secret_scanning@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-google_workspace_transparency: + index_sorting: False + index_template: + index_patterns: + - "logs-google_workspace_transparency-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-google_workspace_transparency@package" + - "logs-google_workspace_transparency@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-google_workspace.admin: + index_sorting: False + index_template: + index_patterns: + - "logs-google_workspace.admin-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-google_workspace.admin@package" + - "logs-google_workspace.admin@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-google_workspace.alert: + index_sorting: False + index_template: + index_patterns: + - "logs-google_workspace.alert-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-google_workspace.alert@package" + - "logs-google_workspace.alert@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-google_workspace.context_aware_access: + index_sorting: False + index_template: + index_patterns: + - "logs-google_workspace.context_aware_access-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-google_workspace.context_aware_access@package" + - "logs-google_workspace.context_aware_access@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-google_workspace.device: + index_sorting: False + index_template: + index_patterns: + - "logs-google_workspace.device-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-google_workspace.device@package" + - "logs-google_workspace.device@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-google_workspace.drive: + index_sorting: False + index_template: + index_patterns: + - "logs-google_workspace.drive-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-google_workspace.drive@package" + - "logs-google_workspace.drive@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-google_workspace.gcp: + index_sorting: False + index_template: + index_patterns: + - "logs-google_workspace.gcp-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-google_workspace.gcp@package" + - "logs-google_workspace.gcp@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-google_workspace.group_enterprise: + index_sorting: False + index_template: + index_patterns: + - "logs-google_workspace.group_enterprise-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-google_workspace.group_enterprise@package" + - "logs-google_workspace.group_enterprise@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-google_workspace.groups: + index_sorting: False + index_template: + index_patterns: + - "logs-google_workspace.groups-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-google_workspace.groups@package" + - "logs-google_workspace.groups@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-google_workspace.login: + index_sorting: False + index_template: + index_patterns: + - "logs-google_workspace.login-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-google_workspace.login@package" + - "logs-google_workspace.login@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-google_workspace.rules: + index_sorting: False + index_template: + index_patterns: + - "logs-google_workspace.rules-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-google_workspace.rules@package" + - "logs-google_workspace.rules@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-google_workspace.saml: + index_sorting: False + index_template: + index_patterns: + - "logs-google_workspace.saml-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-google_workspace.saml@package" + - "logs-google_workspace.saml@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-google_workspace.token: + index_sorting: False + index_template: + index_patterns: + - "logs-google_workspace.token-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-google_workspace.token@package" + - "logs-google_workspace.token@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-google_workspace.user_accounts: + index_sorting: False + index_template: + index_patterns: + - "logs-google_workspace.user_accounts-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-google_workspace.user_accounts@package" + - "logs-google_workspace.user_accounts@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-1password.item_usages: + index_sorting: False + index_template: + index_patterns: + - "logs-1password.item_usages-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-1password.item_usages@package" + - "logs-1password.item_usages@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-1password.signin_attempts: + index_sorting: False + index_template: + index_patterns: + - "logs-1password.signin_attempts-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-1password.signin_attempts@package" + - "logs-1password.signin_attempts@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false so-logs-osquery-manager-actions: index_sorting: False index_template: From a9585b2a7f4afed142fb00a54bf9344a90e199b1 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 23 Jun 2023 10:45:58 -0400 Subject: [PATCH 22/46] Fix Elastic Agent for Heavy --- salt/docker/defaults.yaml | 5 +++++ salt/elasticagent/enabled.sls | 7 ++----- salt/elasticagent/files/elastic-agent.yml.jinja | 7 ++++--- .../elasticagent/tools/sbin/so-elastic-agent-restart | 10 ++++++++++ salt/elasticagent/tools/sbin/so-elastic-agent-start | 12 ++++++++++++ salt/elasticagent/tools/sbin/so-elastic-agent-stop | 12 ++++++++++++ salt/elasticfleet/install_agent_grid.sls | 2 ++ salt/manager/tools/sbin/so-minion | 10 ++++++++++ 8 files changed, 57 insertions(+), 8 deletions(-) create mode 100755 salt/elasticagent/tools/sbin/so-elastic-agent-restart create mode 100755 salt/elasticagent/tools/sbin/so-elastic-agent-start create mode 100755 salt/elasticagent/tools/sbin/so-elastic-agent-stop diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 21b94a2ba..2e5e28a6c 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -178,6 +178,11 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] + 'so-elastic-agent': + final_octet: 46 + custom_bind_mounts: [] + extra_hosts: [] + extra_env: [] 'so-telegraf': final_octet: 99 custom_bind_mounts: [] diff --git a/salt/elasticagent/enabled.sls b/salt/elasticagent/enabled.sls index 52467a54e..ee4b1d2be 100644 --- a/salt/elasticagent/enabled.sls +++ b/salt/elasticagent/enabled.sls @@ -31,19 +31,16 @@ so-elastic-agent: - {{ XTRAHOST }} {% endfor %} {% endif %} - - port_bindings: - {% for BINDING in DOCKER.containers['so-elastic-agent'].port_bindings %} - - {{ BINDING }} - {% endfor %} - binds: - /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro + - /nsm:/nsm:ro {% if DOCKER.containers['so-elastic-agent'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-elastic-agent'].custom_bind_mounts %} - {{ BIND }} {% endfor %} {% endif %} - - environment: {% if DOCKER.containers['so-elastic-agent'].extra_env %} + - environment: {% for XTRAENV in DOCKER.containers['so-elastic-agent'].extra_env %} - {{ XTRAENV }} {% endfor %} diff --git a/salt/elasticagent/files/elastic-agent.yml.jinja b/salt/elasticagent/files/elastic-agent.yml.jinja index 3eb63962b..45bac49b4 100644 --- a/salt/elasticagent/files/elastic-agent.yml.jinja +++ b/salt/elasticagent/files/elastic-agent.yml.jinja @@ -11,6 +11,7 @@ outputs: - 'https://{{ GLOBALS.hostname }}:9200' username: '{{ ES_USER }}' password: '{{ ES_PASS }}' + ssl.verification_mode: none output_permissions: {} agent: download: @@ -29,7 +30,7 @@ inputs: meta: package: name: log - version: 1.1.2 + version: data_stream: namespace: so package_policy_id: 80ffa884-2cfc-459a-964a-34df25714d85 @@ -54,7 +55,7 @@ inputs: meta: package: name: log - version: 1.1.2 + version: data_stream: namespace: so package_policy_id: 90103ac4-f6bd-4a4a-b596-952c332390fc @@ -79,7 +80,7 @@ inputs: meta: package: name: log - version: 1.1.2 + version: data_stream: namespace: so package_policy_id: 6197fe84-9b58-4d9b-8464-3d517f28808d diff --git a/salt/elasticagent/tools/sbin/so-elastic-agent-restart b/salt/elasticagent/tools/sbin/so-elastic-agent-restart new file mode 100755 index 000000000..cb125ce10 --- /dev/null +++ b/salt/elasticagent/tools/sbin/so-elastic-agent-restart @@ -0,0 +1,10 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +/usr/sbin/so-restart elastic-agent $1 diff --git a/salt/elasticagent/tools/sbin/so-elastic-agent-start b/salt/elasticagent/tools/sbin/so-elastic-agent-start new file mode 100755 index 000000000..294b9ba7a --- /dev/null +++ b/salt/elasticagent/tools/sbin/so-elastic-agent-start @@ -0,0 +1,12 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-common + +/usr/sbin/so-start elastic-agent $1 diff --git a/salt/elasticagent/tools/sbin/so-elastic-agent-stop b/salt/elasticagent/tools/sbin/so-elastic-agent-stop new file mode 100755 index 000000000..cbfdcd380 --- /dev/null +++ b/salt/elasticagent/tools/sbin/so-elastic-agent-stop @@ -0,0 +1,12 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-common + +/usr/sbin/so-stop elastic-agent $1 diff --git a/salt/elasticfleet/install_agent_grid.sls b/salt/elasticfleet/install_agent_grid.sls index 00a74437e..2199d2530 100644 --- a/salt/elasticfleet/install_agent_grid.sls +++ b/salt/elasticfleet/install_agent_grid.sls @@ -7,10 +7,12 @@ {% set AGENT_STATUS = salt['service.available']('elastic-agent') %} {% if not AGENT_STATUS %} +{% if grains.role not in ['so-heavy'] %} run_installer: cmd.script: - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 - cwd: /opt/so - args: -token={{ GRIDNODETOKEN }} +{% endif %} {% endif %} diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index e9aff9e09..df0adb73c 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -133,6 +133,15 @@ function add_elasticsearch_to_minion() { " " >> $PILLARFILE } + +# Add Elastic Agent settings to the minion file +function add_elastic_agent_to_minion() { + printf '%s\n'\ + "elasticagent:"\ + " enabled: True"\ + " " >> $PILLARFILE +} + # Add Elastic Fleet Server settings to the minion file function add_fleet_to_minion() { @@ -518,6 +527,7 @@ function createIDH() { function createHEAVYNODE() { add_elasticsearch_to_minion + add_elastic_agent_to_minion add_logstash_to_minion add_sensor_to_minion add_strelka_to_minion From 261acee8a0a5d884f25b8e2e09127932bb41a0ca Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 20 Jun 2023 13:15:15 -0600 Subject: [PATCH 23/46] New Hunt queryToggleFilter New filter to exclude soc logs from hunt results. --- salt/soc/defaults.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 156446b7f..2e7bdcaf0 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1117,6 +1117,9 @@ soc: - name: caseExcludeToggle filter: 'NOT _index:"*:so-case*"' enabled: true + - name: socExcludeToggle + filter: 'NOT event.module:"soc"' + enabled: true queries: - name: Default Query description: Show all events grouped by the observer host From fb27e7c479b1d25712545ce139d8f19e09dd7376 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 22 Jun 2023 17:29:11 -0600 Subject: [PATCH 24/46] Also add to dashboard Duplicate new queryToggleFilter from hunt to dashboard. --- salt/soc/defaults.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 2e7bdcaf0..81c334d32 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1387,6 +1387,9 @@ soc: - name: caseExcludeToggle filter: 'NOT _index:"*:so-case*"' enabled: true + - name: socExcludeToggle + filter: 'NOT event.module:"soc"' + enabled: true queries: - name: Overview description: Overview of all events From b96d3473f222c4bf6c30194328912cffc1ee85c4 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 23 Jun 2023 18:38:04 +0000 Subject: [PATCH 25/46] Fix indentation --- salt/elasticsearch/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 5442969e5..f53e82de1 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -314,7 +314,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.ec2_logs: + so-logs-aws.ec2_logs: index_sorting: False index_template: index_patterns: From ff7db0be6394ba0b0ee623a3154f3aa7086e3422 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 23 Jun 2023 15:31:11 -0400 Subject: [PATCH 26/46] Remove old index settings --- setup/so-functions | 41 +---------------------------------------- 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index ef4e44eaa..72b7d2c2f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1149,46 +1149,7 @@ elasticsearch_pillar() { " indices:"\ " query:"\ " bool:"\ - " max_clause_count: 3500"\ - " index_settings:"\ > $elasticsearch_pillar_file - for INDEX in aws azure barracuda beats bluecoat cef checkpoint cisco cyberark cylance elasticsearch endgame f5 firewall fortinet gcp google_workspace imperva infoblox juniper kibana logstash microsoft misp netflow netscout o365 okta osquery proofpoint radware redis snort snyk sonicwall sophos strelka syslog tomcat zeek zscaler - do - printf '%s\n'\ - " so-$INDEX:"\ - " warm: 7"\ - " close: 30"\ - " delete: 365"\ - " index_sorting: False"\ - " index_template:"\ - " template:"\ - " settings:"\ - " index:"\ - " mapping:"\ - " total_fields:"\ - " limit: 5000"\ - " refresh_interval: 30s"\ - " number_of_shards: 1"\ - " number_of_replicas: 0" >> $elasticsearch_pillar_file - done - for INDEX in import - do - printf '%s\n'\ - " so-$INDEX:"\ - " warm: 7"\ - " close: 73000"\ - " delete: 73001"\ - " index_sorting: False"\ - " index_template:"\ - " template:"\ - " settings:"\ - " index:"\ - " mapping:"\ - " total_fields:"\ - " limit: 5000"\ - " refresh_interval: 30s"\ - " number_of_shards: 1"\ - " number_of_replicas: 0" >> $elasticsearch_pillar_file - done + " max_clause_count: 3500" > $elasticsearch_pillar_file } es_heapsize() { From bb0cfc52538fdaad88f401eff2447c4dfc695ac4 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 23 Jun 2023 15:49:03 -0400 Subject: [PATCH 27/46] Create & assign Heavy Node Fleet Policy --- pillar/top.sls | 2 - .../elasticsearch-logs.json | 106 ++++++++++++++++++ .../idh-logs.json | 2 +- .../import-evtx-logs.json | 2 +- .../import-suricata-logs.json | 2 +- .../grid-nodes_general/kratos-logs.json | 29 +++++ .../osquery-grid-nodes.json | 20 ++++ .../grid-nodes_general/redis-logs.json | 76 +++++++++++++ .../soc-auth-sync-logs.json | 29 +++++ .../soc-salt-relay-logs.json | 29 +++++ .../soc-sensoroni-logs.json | 29 +++++ .../grid-nodes_general/soc-server-logs.json | 29 +++++ .../strelka-logs.json | 2 +- .../suricata-logs.json | 2 +- .../syslog-tcp-514.json | 2 +- .../syslog-udp-514.json | 2 +- .../grid-nodes_general/system-grid-nodes.json | 40 +++++++ .../elasticsearch-logs.json | 2 +- .../kratos-logs.json | 2 +- .../osquery-grid-nodes.json | 2 +- .../redis-logs.json | 2 +- .../soc-auth-sync-logs.json | 2 +- .../soc-salt-relay-logs.json | 2 +- .../soc-sensoroni-logs.json | 2 +- .../soc-server-logs.json | 2 +- .../system-grid-nodes.json | 2 +- salt/elasticfleet/install_agent_grid.sls | 13 ++- .../so-elastic-fleet-integration-policy-load | 27 ++++- .../tools/sbin_jinja/so-elastic-fleet-setup | 16 ++- 29 files changed, 447 insertions(+), 30 deletions(-) create mode 100644 salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_general}/idh-logs.json (97%) rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_general}/import-evtx-logs.json (98%) rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_general}/import-suricata-logs.json (95%) create mode 100644 salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_general}/strelka-logs.json (94%) rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_general}/suricata-logs.json (94%) rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_general}/syslog-tcp-514.json (94%) rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_general}/syslog-udp-514.json (95%) create mode 100644 salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_heavy}/elasticsearch-logs.json (98%) rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_heavy}/kratos-logs.json (95%) rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_heavy}/osquery-grid-nodes.json (89%) rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_heavy}/redis-logs.json (97%) rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_heavy}/soc-auth-sync-logs.json (95%) rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_heavy}/soc-salt-relay-logs.json (95%) rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_heavy}/soc-sensoroni-logs.json (97%) rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_heavy}/soc-server-logs.json (97%) rename salt/elasticfleet/files/integrations/{grid-nodes => grid-nodes_heavy}/system-grid-nodes.json (95%) diff --git a/pillar/top.sls b/pillar/top.sls index 464307065..75117e35f 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -189,8 +189,6 @@ base: - logstash.adv_logstash - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch - - elasticagent.soc_elasticagent - - elasticagent.adv_elasticagent - curator.soc_curator - curator.adv_curator - redis.soc_redis diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json new file mode 100644 index 000000000..4c22f92ee --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json @@ -0,0 +1,106 @@ +{ + "package": { + "name": "elasticsearch", + "version": "" + }, + "name": "elasticsearch-logs", + "namespace": "default", + "description": "Elasticsearch Logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "elasticsearch-logfile": { + "enabled": true, + "streams": { + "elasticsearch.audit": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/*_audit.json" + ] + } + }, + "elasticsearch.deprecation": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/*_deprecation.json" + ] + } + }, + "elasticsearch.gc": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/gc.log.[0-9]*", + "/var/log/elasticsearch/gc.log" + ] + } + }, + "elasticsearch.server": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/elasticsearch/*.log" + ] + } + }, + "elasticsearch.slowlog": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/*_index_search_slowlog.json", + "/var/log/elasticsearch/*_index_indexing_slowlog.json" + ] + } + } + } + }, + "elasticsearch-elasticsearch/metrics": { + "enabled": false, + "vars": { + "hosts": [ + "http://localhost:9200" + ], + "scope": "node" + }, + "streams": { + "elasticsearch.stack_monitoring.ccr": { + "enabled": false + }, + "elasticsearch.stack_monitoring.cluster_stats": { + "enabled": false + }, + "elasticsearch.stack_monitoring.enrich": { + "enabled": false + }, + "elasticsearch.stack_monitoring.index": { + "enabled": false + }, + "elasticsearch.stack_monitoring.index_recovery": { + "enabled": false, + "vars": { + "active.only": true + } + }, + "elasticsearch.stack_monitoring.index_summary": { + "enabled": false + }, + "elasticsearch.stack_monitoring.ml_job": { + "enabled": false + }, + "elasticsearch.stack_monitoring.node": { + "enabled": false + }, + "elasticsearch.stack_monitoring.node_stats": { + "enabled": false + }, + "elasticsearch.stack_monitoring.pending_tasks": { + "enabled": false + }, + "elasticsearch.stack_monitoring.shard": { + "enabled": false + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/idh-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json similarity index 97% rename from salt/elasticfleet/files/integrations/grid-nodes/idh-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json index 95b72e0a0..32055112a 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/idh-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json @@ -6,7 +6,7 @@ "name": "idh-logs", "namespace": "so", "description": "IDH integration", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json similarity index 98% rename from salt/elasticfleet/files/integrations/grid-nodes/import-evtx-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 5bebfd54d..d9f8daeb9 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -6,7 +6,7 @@ "name": "import-evtx-logs", "namespace": "so", "description": "Import Windows EVTX logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "vars": {}, "inputs": { "logs-logfile": { diff --git a/salt/elasticfleet/files/integrations/grid-nodes/import-suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json similarity index 95% rename from salt/elasticfleet/files/integrations/grid-nodes/import-suricata-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json index 4cba27121..f17ee33d1 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/import-suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json @@ -6,7 +6,7 @@ "name": "import-suricata-logs", "namespace": "so", "description": "Import Suricata logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json new file mode 100644 index 000000000..c342b57bd --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "" + }, + "name": "kratos-logs", + "namespace": "so", + "description": "Kratos logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/kratos/kratos.log" + ], + "data_stream.dataset": "kratos", + "tags": ["so-kratos"], + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos", + "custom": "pipeline: kratos" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json new file mode 100644 index 000000000..0349c9fc3 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json @@ -0,0 +1,20 @@ +{ + "package": { + "name": "osquery_manager", + "version": "" + }, + "name": "osquery-grid-nodes", + "namespace": "default", + "policy_id": "so-grid-nodes_general", + "inputs": { + "osquery_manager-osquery": { + "enabled": true, + "streams": { + "osquery_manager.result": { + "enabled": true, + "vars": {} + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json new file mode 100644 index 000000000..6b9cbffaf --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json @@ -0,0 +1,76 @@ +{ + "package": { + "name": "redis", + "version": "" + }, + "name": "redis-logs", + "namespace": "default", + "description": "Redis logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "redis-logfile": { + "enabled": true, + "streams": { + "redis.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/redis/redis.log" + ], + "tags": [ + "redis-log" + ], + "preserve_original_event": false + } + } + } + }, + "redis-redis": { + "enabled": false, + "streams": { + "redis.slowlog": { + "enabled": false, + "vars": { + "hosts": [ + "127.0.0.1:6379" + ], + "password": "" + } + } + } + }, + "redis-redis/metrics": { + "enabled": false, + "vars": { + "hosts": [ + "127.0.0.1:6379" + ], + "idle_timeout": "20s", + "maxconn": 10, + "network": "tcp", + "password": "" + }, + "streams": { + "redis.info": { + "enabled": false, + "vars": { + "period": "10s" + } + }, + "redis.key": { + "enabled": false, + "vars": { + "key.patterns": "- limit: 20\n pattern: *\n", + "period": "10s" + } + }, + "redis.keyspace": { + "enabled": false, + "vars": { + "period": "10s" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json new file mode 100644 index 000000000..84e9ae94d --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "" + }, + "name": "soc-auth-sync-logs", + "namespace": "so", + "description": "Security Onion - Elastic Auth Sync - Logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/soc/sync.log" + ], + "data_stream.dataset": "soc", + "tags": ["so-soc"], + "processors": "- dissect:\n tokenizer: \"%{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: auth_sync", + "custom": "pipeline: common" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json new file mode 100644 index 000000000..07bd89b89 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "" + }, + "name": "soc-salt-relay-logs", + "namespace": "so", + "description": "Security Onion - Salt Relay - Logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/soc/salt-relay.log" + ], + "data_stream.dataset": "soc", + "tags": ["so-soc"], + "processors": "- dissect:\n tokenizer: \"%{soc.ts} | %{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: salt_relay", + "custom": "pipeline: common" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json new file mode 100644 index 000000000..bee14ebf5 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "" + }, + "name": "soc-sensoroni-logs", + "namespace": "so", + "description": "Security Onion - Sensoroni - Logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/sensoroni/sensoroni.log" + ], + "data_stream.dataset": "soc", + "tags": [], + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"sensoroni\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: sensoroni\n- rename:\n fields:\n - from: \"sensoroni.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"sensoroni.fields.status\"\n to: \"http.response.status_code\"\n - from: \"sensoroni.fields.method\"\n to: \"http.request.method\"\n - from: \"sensoroni.fields.path\"\n to: \"url.path\"\n - from: \"sensoroni.message\"\n to: \"event.action\"\n - from: \"sensoroni.level\"\n to: \"log.level\"\n ignore_missing: true", + "custom": "pipeline: common" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json new file mode 100644 index 000000000..285d79148 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "" + }, + "name": "soc-server-logs", + "namespace": "so", + "description": "Security Onion Console Logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/soc/sensoroni-server.log" + ], + "data_stream.dataset": "soc", + "tags": ["so-soc"], + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"soc\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: server\n- rename:\n fields:\n - from: \"soc.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"soc.fields.status\"\n to: \"http.response.status_code\"\n - from: \"soc.fields.method\"\n to: \"http.request.method\"\n - from: \"soc.fields.path\"\n to: \"url.path\"\n - from: \"soc.message\"\n to: \"event.action\"\n - from: \"soc.level\"\n to: \"log.level\"\n ignore_missing: true", + "custom": "pipeline: common" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/strelka-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json similarity index 94% rename from salt/elasticfleet/files/integrations/grid-nodes/strelka-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json index ac6157638..6f6beca99 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/strelka-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json @@ -6,7 +6,7 @@ "name": "strelka-logs", "namespace": "so", "description": "Strelka logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json similarity index 94% rename from salt/elasticfleet/files/integrations/grid-nodes/suricata-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json index 9d7e4040d..7ff43c3a8 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json @@ -6,7 +6,7 @@ "name": "suricata-logs", "namespace": "so", "description": "Suricata integration", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/syslog-tcp-514.json b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json similarity index 94% rename from salt/elasticfleet/files/integrations/grid-nodes/syslog-tcp-514.json rename to salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json index 495aaa309..80baa45ca 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/syslog-tcp-514.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json @@ -6,7 +6,7 @@ "name": "syslog-tcp-514", "namespace": "so", "description": "Syslog Over TCP Port 514", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "inputs": { "tcp-tcp": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/syslog-udp-514.json b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json similarity index 95% rename from salt/elasticfleet/files/integrations/grid-nodes/syslog-udp-514.json rename to salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json index 053e95299..653c788b5 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/syslog-udp-514.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json @@ -6,7 +6,7 @@ "name": "syslog-udp-514", "namespace": "so", "description": "Syslog over UDP Port 514", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "inputs": { "udp-udp": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json new file mode 100644 index 000000000..a5c4c3e81 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json @@ -0,0 +1,40 @@ +{ + "policy_id": "so-grid-nodes_general", + "package": { + "name": "system", + "version": "" + }, + "name": "system-grid-nodes", + "namespace": "default", + "inputs": { + "system-logfile": { + "enabled": true, + "streams": { + "system.auth": { + "enabled": true, + "vars": { + "paths": [ + "/var/log/auth.log*", + "/var/log/secure*" + ] + } + }, + "system.syslog": { + "enabled": true, + "vars": { + "paths": [ + "/var/log/messages*", + "/var/log/syslog*" + ] + } + } + } + }, + "system-winlog": { + "enabled": false + }, + "system-system/metrics": { + "enabled": false + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/elasticsearch-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-logs.json similarity index 98% rename from salt/elasticfleet/files/integrations/grid-nodes/elasticsearch-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-logs.json index 99f2733c9..711602775 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/elasticsearch-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-logs.json @@ -6,7 +6,7 @@ "name": "elasticsearch-logs", "namespace": "default", "description": "Elasticsearch Logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "elasticsearch-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json similarity index 95% rename from salt/elasticfleet/files/integrations/grid-nodes/kratos-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json index 5e134f1f6..c9e4183de 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/kratos-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json @@ -6,7 +6,7 @@ "name": "kratos-logs", "namespace": "so", "description": "Kratos logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/osquery-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json similarity index 89% rename from salt/elasticfleet/files/integrations/grid-nodes/osquery-grid-nodes.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json index 197526ce3..d0281c111 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/osquery-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json @@ -5,7 +5,7 @@ }, "name": "osquery-grid-nodes", "namespace": "default", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "osquery_manager-osquery": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/redis-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/redis-logs.json similarity index 97% rename from salt/elasticfleet/files/integrations/grid-nodes/redis-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/redis-logs.json index a5d4102df..cddcedfd8 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/redis-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/redis-logs.json @@ -6,7 +6,7 @@ "name": "redis-logs", "namespace": "default", "description": "Redis logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "redis-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json similarity index 95% rename from salt/elasticfleet/files/integrations/grid-nodes/soc-auth-sync-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json index 7f60d1706..2004c8c5d 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/soc-auth-sync-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json @@ -6,7 +6,7 @@ "name": "soc-auth-sync-logs", "namespace": "so", "description": "Security Onion - Elastic Auth Sync - Logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json similarity index 95% rename from salt/elasticfleet/files/integrations/grid-nodes/soc-salt-relay-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json index 7821f4081..b1b6098c1 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/soc-salt-relay-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json @@ -6,7 +6,7 @@ "name": "soc-salt-relay-logs", "namespace": "so", "description": "Security Onion - Salt Relay - Logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json similarity index 97% rename from salt/elasticfleet/files/integrations/grid-nodes/soc-sensoroni-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json index 56069ed65..5954e5052 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/soc-sensoroni-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json @@ -6,7 +6,7 @@ "name": "soc-sensoroni-logs", "namespace": "so", "description": "Security Onion - Sensoroni - Logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json similarity index 97% rename from salt/elasticfleet/files/integrations/grid-nodes/soc-server-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json index fcdfc9344..89e26563a 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/soc-server-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json @@ -6,7 +6,7 @@ "name": "soc-server-logs", "namespace": "so", "description": "Security Onion Console Logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json similarity index 95% rename from salt/elasticfleet/files/integrations/grid-nodes/system-grid-nodes.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json index 3c10227ca..31d30d4e0 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json @@ -1,5 +1,5 @@ { - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "package": { "name": "system", "version": "" diff --git a/salt/elasticfleet/install_agent_grid.sls b/salt/elasticfleet/install_agent_grid.sls index 2199d2530..1d6c5a241 100644 --- a/salt/elasticfleet/install_agent_grid.sls +++ b/salt/elasticfleet/install_agent_grid.sls @@ -2,17 +2,24 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. -{%- set GRIDNODETOKEN = salt['pillar.get']('global:fleet_grid_enrollment_token') -%} +{%- set GRIDNODETOKENGENERAL = salt['pillar.get']('global:fleet_grid_enrollment_token_general') -%} +{%- set GRIDNODETOKENHEAVY = salt['pillar.get']('global:fleet_grid_enrollment_token_heavy') -%} {% set AGENT_STATUS = salt['service.available']('elastic-agent') %} {% if not AGENT_STATUS %} -{% if grains.role not in ['so-heavy'] %} +{% if grains.role not in ['so-heavynode'] %} run_installer: cmd.script: - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 - cwd: /opt/so - - args: -token={{ GRIDNODETOKEN }} + - args: -token={{ GRIDNODETOKENGENERAL }} +{% else %} +run_installer: + cmd.script: + - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 + - cwd: /opt/so + - args: -token={{ GRIDNODETOKENHEAVY }} {% endif %} {% endif %} diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index 771d923ef..54538ed9e 100755 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -25,11 +25,30 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then fi done - # Grid Nodes - for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes/*.json + # Grid Nodes - General + for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json do - printf "\n\nGrid Nodes Policy - Loading $INTEGRATION\n" - elastic_fleet_integration_check "so-grid-nodes" "$INTEGRATION" + printf "\n\nGrid Nodes Policy_General - Loading $INTEGRATION\n" + elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION" + if [ -n "$INTEGRATION_ID" ]; then + printf "\n\nIntegration $NAME exists - Updating integration\n" + elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION" + else + printf "\n\nIntegration does not exist - Creating integration\n" + if [ "$NAME" != "elasticsearch-logs" ]; then + elastic_fleet_integration_create "@$INTEGRATION" + fi + fi + done + if [[ "$RETURN_CODE" != "1" ]]; then + touch /opt/so/state/eaintegrations.txt + fi + + # Grid Nodes - Heavy + for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json + do + printf "\n\nGrid Nodes Policy_Heavy - Loading $INTEGRATION\n" + elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION" if [ -n "$INTEGRATION_ID" ]; then printf "\n\nIntegration $NAME exists - Updating integration\n" elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION" diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 94a42a70a..3b75fef56 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -62,8 +62,11 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fle # Initial Endpoints Policy elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600" -# Grid Nodes Policy -elastic_fleet_policy_create "so-grid-nodes" "SO Grid Node Policy" "false" "1209600" +# Grid Nodes - General Policy +elastic_fleet_policy_create "so-grid-nodes_general" "SO Grid Nodes - General Purpose" "false" "1209600" + +# Grid Nodes - Heavy Node Policy +elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy Node" "false" "1209600" # Load Integrations for default policies so-elastic-fleet-integration-policy-load @@ -81,7 +84,8 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl # Query for Enrollment Tokens for default policies ENDPOINTSENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') -GRIDNODESENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes")) | .api_key') +GRIDNODESENROLLMENTOKENGENERAL=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key') +GRIDNODESENROLLMENTOKENHEAVY=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key') # Store needed data in minion pillar pillar_file=/opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls @@ -92,13 +96,15 @@ printf '%s\n'\ " server:"\ " es_token: '$ESTOKEN'"\ " endpoints_enrollment: '$ENDPOINTSENROLLMENTOKEN'"\ - " grid_enrollment: '$GRIDNODESENROLLMENTOKEN'"\ + " grid_enrollment_general: '$GRIDNODESENROLLMENTOKENGENERAL'"\ + " grid_enrollment_heavy: '$GRIDNODESENROLLMENTOKENHEAVY'"\ "" >> "$pillar_file" #Store Grid Nodes Enrollment token in Global pillar global_pillar_file=/opt/so/saltstack/local/pillar/global/soc_global.sls printf '%s\n'\ - " fleet_grid_enrollment_token: '$GRIDNODESENROLLMENTOKEN'"\ + " fleet_grid_enrollment_token_general: '$GRIDNODESENROLLMENTOKENGENERAL'"\ + " fleet_grid_enrollment_token_heavy: '$GRIDNODESENROLLMENTOKENHEAVY'"\ "" >> "$global_pillar_file" # Call Elastic-Fleet Salt State From f63c26b7f2e5640848166a14386503db339a3965 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 23 Jun 2023 15:50:54 -0400 Subject: [PATCH 28/46] soup fix --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 516facae4..35607824a 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -178,7 +178,7 @@ update_registry() { check_airgap() { # See if this is an airgap install - AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}') + AIRGAP=$(cat /opt/so/saltstack/local/pillar/global/soc_global.sls | grep airgap: | awk '{print $2}') if [[ "$AIRGAP" == "True" ]]; then is_airgap=0 UPDATE_DIR=/tmp/soagupdate/SecurityOnion From 801a5a6824206a75c7ced4c3abbf87d4798c3caf Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 23 Jun 2023 15:56:15 -0400 Subject: [PATCH 29/46] soup fix --- salt/common/soup_scripts.sls | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index ed3b8696c..8dff85ddb 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -8,6 +8,15 @@ soup_scripts: - source: salt://common/tools/sbin - include_pat: - so-common - - so-firewall - so-image-common - - soup + +soup_manager_scripts: + file.recurse: + - name: /usr/sbin + - user: root + - group: root + - file_mode: 755 + - source: salt://manager/tools/sbin + - include_pat: + - so-firewall + - soup \ No newline at end of file From aa91c1fef2af9725bf01818936b381c1de39b940 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 23 Jun 2023 15:57:30 -0400 Subject: [PATCH 30/46] Add empty object for index_settings --- setup/so-functions | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 72b7d2c2f..8b65d97e3 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1149,7 +1149,8 @@ elasticsearch_pillar() { " indices:"\ " query:"\ " bool:"\ - " max_clause_count: 3500" > $elasticsearch_pillar_file + " max_clause_count: 3500"\ + " index_settings: {}" > $elasticsearch_pillar_file } es_heapsize() { From 25b51135fc529846e2b7b1d1e82aaddd7b4ea502 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 23 Jun 2023 15:58:20 -0400 Subject: [PATCH 31/46] soup fix --- salt/manager/tools/sbin/soup | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 35607824a..35d5ea731 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -583,22 +583,22 @@ upgrade_salt() { verify_latest_update_script() { # Check to see if the update scripts match. If not run the new one. CURRENTSOUP=$(md5sum /usr/sbin/soup | awk '{print $1}') - GITSOUP=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/soup | awk '{print $1}') + GITSOUP=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/soup | awk '{print $1}') CURRENTCMN=$(md5sum /usr/sbin/so-common | awk '{print $1}') - GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}') + GITCMN=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-common | awk '{print $1}') CURRENTIMGCMN=$(md5sum /usr/sbin/so-image-common | awk '{print $1}') - GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}') + GITIMGCMN=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-image-common | awk '{print $1}') CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}') - GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-firewall | awk '{print $1}') + GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-firewall | awk '{print $1}') if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then echo "This version of the soup script is up to date. Proceeding." else echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete." - cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/ + cp $UPDATE_DIR/salt/manager/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - cp $UPDATE_DIR/salt/common/tools/sbin/so-firewall $DEFAULT_SALT_DIR/salt/common/tools/sbin/ + cp $UPDATE_DIR/salt/manager/tools/sbin/so-firewall $DEFAULT_SALT_DIR/salt/common/tools/sbin/ salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local echo "" echo "The soup script has been modified. Please run soup again to continue the upgrade." From 5da5a04025149d84173badabeaa6369842bfc8e5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 23 Jun 2023 16:00:02 -0400 Subject: [PATCH 32/46] soup fix --- salt/manager/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 35d5ea731..0ab2d6b7d 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -585,9 +585,9 @@ verify_latest_update_script() { CURRENTSOUP=$(md5sum /usr/sbin/soup | awk '{print $1}') GITSOUP=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/soup | awk '{print $1}') CURRENTCMN=$(md5sum /usr/sbin/so-common | awk '{print $1}') - GITCMN=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-common | awk '{print $1}') + GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}') CURRENTIMGCMN=$(md5sum /usr/sbin/so-image-common | awk '{print $1}') - GITIMGCMN=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-image-common | awk '{print $1}') + GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}') CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}') GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-firewall | awk '{print $1}') From 43832f9c34c7a43cef13ddea161c841b75118022 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 23 Jun 2023 16:03:51 -0400 Subject: [PATCH 33/46] soup fix --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0ab2d6b7d..5dd4e3888 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -746,7 +746,7 @@ main() { stop_salt_master - update_repo + #update_repo # Does salt need upgraded. If so update it. if [[ $UPGRADESALT -eq 1 ]]; then From e2ff48164bf30176239b7899c628cec5b9ad162a Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 23 Jun 2023 16:03:58 -0400 Subject: [PATCH 34/46] Only load if so-elastic-fleet-common exists --- .../so-elasticsearch-templates-load | 114 +++++++++--------- 1 file changed, 59 insertions(+), 55 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 60e32c12b..991f54819 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -15,62 +15,66 @@ default_conf_dir=/opt/so/conf # Define a default directory to load pipelines from ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/" -# Wait for ElasticSearch to initialize -echo -n "Waiting for ElasticSearch..." -COUNT=0 -ELASTICSEARCH_CONNECTED="no" -while [[ "$COUNT" -le 240 ]]; do - so-elasticsearch-query / -k --output /dev/null --silent --head --fail - if [ $? -eq 0 ]; then - ELASTICSEARCH_CONNECTED="yes" - echo "connected!" - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." +if [ -f /usr/sbin/so-elastic-fleet-common ]; then + # Wait for ElasticSearch to initialize + echo -n "Waiting for ElasticSearch..." + COUNT=0 + ELASTICSEARCH_CONNECTED="no" + while [[ "$COUNT" -le 240 ]]; do + so-elasticsearch-query / -k --output /dev/null --silent --head --fail + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi + done + if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo + exit 1 fi -done -if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + + SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') + INSTALLED=$(elastic_fleet_package_is_installed {{ SUPPORTED_PACKAGES[0] }} ) + if [ "$INSTALLED" != "installed" ]; then + echo + echo "Packages not yet installed." + echo + exit 0 + fi + + set -e + + cd ${ELASTICSEARCH_TEMPLATES}/component/ecs + + echo "Loading ECS component templates..." + for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE-mappings"; so-elasticsearch-query _component_template/$TEMPLATE-mappings -d@$i -XPUT 2>/dev/null; echo; done + + cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent + + echo "Loading Elastic Agent component templates..." + for i in *; do TEMPLATE=${i::-5}; echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done + + # Load SO-specific component templates + cd ${ELASTICSEARCH_TEMPLATES}/component/so + + echo "Loading Security Onion component templates..." + for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done echo - echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + + # Load SO index templates + cd ${ELASTICSEARCH_TEMPLATES}/index + + echo "Loading Security Onion index templates..." + for i in *; do TEMPLATE=${i::-14}; echo "$TEMPLATE"; so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done echo - exit 1 +else + echo "Elastic Fleet not configured. Exiting..." + exit 0 fi - -SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') -INSTALLED=$(elastic_fleet_package_is_installed {{ SUPPORTED_PACKAGES[0] }} ) -if [ "$INSTALLED" != "installed" ]; then - echo - echo "Packages not yet installed." - echo - exit 0 -fi - -set -e - -cd ${ELASTICSEARCH_TEMPLATES}/component/ecs - -echo "Loading ECS component templates..." -for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE-mappings"; so-elasticsearch-query _component_template/$TEMPLATE-mappings -d@$i -XPUT 2>/dev/null; echo; done - -cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent - -echo "Loading Elastic Agent component templates..." -for i in *; do TEMPLATE=${i::-5}; echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done - -# Load SO-specific component templates -cd ${ELASTICSEARCH_TEMPLATES}/component/so - -echo "Loading Security Onion component templates..." -for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done -echo - -# Load SO index templates -cd ${ELASTICSEARCH_TEMPLATES}/index - -echo "Loading Security Onion index templates..." -for i in *; do TEMPLATE=${i::-14}; echo "$TEMPLATE"; so-elasticsearch-query _index_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done -echo - -cd - >/dev/null + cd - >/dev/null From 5073d62ee858ad3b4accd477ea4436bd4ede496f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 23 Jun 2023 16:09:57 -0400 Subject: [PATCH 35/46] soup fix --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 5dd4e3888..d044355e0 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -303,7 +303,7 @@ check_log_size_limit() { check_os_updates() { # Check to see if there are OS updates NEEDUPDATES="We have detected missing operating system (OS) updates. Do you want to install these OS updates now? This could take a while depending on the size of your grid and how many packages are missing, but it is recommended to keep your system updated." - OSUPDATES=$(yum -q list updates | wc -l) + OSUPDATES=$(dnf -q list updates | grep -v docker | grep -v containerd | grep -c salt | wc -l) if [[ "$OSUPDATES" -gt 0 ]]; then if [[ -z $UNATTENDED ]]; then echo "$NEEDUPDATES" From e02bdffe3418c68ab50fc9d7814e9ade7dd8b646 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 23 Jun 2023 16:10:22 -0400 Subject: [PATCH 36/46] Fix typos --- salt/elasticsearch/defaults.yaml | 40 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index f53e82de1..3f29483e0 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -386,18 +386,18 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.route53.resolver_logs: + so-logs-aws.route53_resolver_logs: index_sorting: False index_template: index_patterns: - - "logs-aws.route53.resolver_logs-*" + - "logs-aws.route53_resolver_logs-*" template: settings: index: number_of_replicas: 0 composed_of: - - "logs-aws.route53.resolver_logs@package" - - "logs-aws.route53.resolver_logs@custom" + - "logs-aws.route53_resolver_logs@package" + - "logs-aws.route53_resolver_logs@custom" - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 501 @@ -548,18 +548,18 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.identity_protect: + so-logs-azure.identity_protection: index_sorting: False index_template: index_patterns: - - "logs-azure.identity_protect-*" + - "logs-azure.identity_protection-*" template: settings: index: number_of_replicas: 0 composed_of: - - "logs-azure.identity_protect@package" - - "logs-azure.identity_protect@custom" + - "logs-azure.identity_protection@package" + - "logs-azure.identity_protection@custom" - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 501 @@ -638,36 +638,36 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-cloudflare-audit: + so-logs-cloudflare.audit: index_sorting: False index_template: index_patterns: - - "logs-cloudflare-audit-*" + - "logs-cloudflare.audit-*" template: settings: index: number_of_replicas: 0 composed_of: - - "logs-cloudflare-audit@package" - - "logs-cloudflare-audit@custom" + - "logs-cloudflare.audit@package" + - "logs-cloudflare.audit@custom" - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 501 data_stream: hidden: false allow_custom_routing: false - so-logs-cloudflare-logpull: + so-logs-cloudflare.logpull: index_sorting: False index_template: index_patterns: - - "logs-cloudflare-logpull-*" + - "logs-cloudflare.logpull-*" template: settings: index: number_of_replicas: 0 composed_of: - - "logs-cloudflare-logpull@package" - - "logs-cloudflare-logpull@custom" + - "logs-cloudflare.logpull@package" + - "logs-cloudflare.logpull@custom" - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 501 @@ -782,18 +782,18 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace_transparency: + so-logs-google_workspace.access_transparency: index_sorting: False index_template: index_patterns: - - "logs-google_workspace_transparency-*" + - "logs-google_workspace.access_transparency-*" template: settings: index: number_of_replicas: 0 composed_of: - - "logs-google_workspace_transparency@package" - - "logs-google_workspace_transparency@custom" + - "logs-google_workspace.access_transparency@package" + - "logs-google_workspace.access_transparency@custom" - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 501 From 3018886f727338f2127d886e38537d93df27ec2d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 23 Jun 2023 16:13:04 -0400 Subject: [PATCH 37/46] soup fix --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d044355e0..168da7417 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -303,7 +303,7 @@ check_log_size_limit() { check_os_updates() { # Check to see if there are OS updates NEEDUPDATES="We have detected missing operating system (OS) updates. Do you want to install these OS updates now? This could take a while depending on the size of your grid and how many packages are missing, but it is recommended to keep your system updated." - OSUPDATES=$(dnf -q list updates | grep -v docker | grep -v containerd | grep -c salt | wc -l) + OSUPDATES=$(dnf -q list updates | grep -v docker | grep -v containerd | grep -v salt | grep -v Available | wc -l) if [[ "$OSUPDATES" -gt 0 ]]; then if [[ -z $UNATTENDED ]]; then echo "$NEEDUPDATES" From 0607532e4a7989e799feb1190df33ca342a6c30d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 23 Jun 2023 16:20:41 -0400 Subject: [PATCH 38/46] soup fix --- salt/manager/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 168da7417..f48334b02 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -7,6 +7,7 @@ . /usr/sbin/so-common +. /usr/sbin/so-image-common UPDATE_DIR=/tmp/sogh/securityonion DEFAULT_SALT_DIR=/opt/so/saltstack/default From 05b84327b847c296d19f0a2f2090fcfa1e0fe299 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 23 Jun 2023 16:32:57 -0400 Subject: [PATCH 39/46] Fix typo --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 8b65d97e3..baeac99f0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1150,7 +1150,7 @@ elasticsearch_pillar() { " query:"\ " bool:"\ " max_clause_count: 3500"\ - " index_settings: {}" > $elasticsearch_pillar_file + " index_settings: {}" > $elasticsearch_pillar_file } es_heapsize() { From ec8eac3430895f120c076ed7de409b22a3b4a2fe Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 23 Jun 2023 17:05:41 -0400 Subject: [PATCH 40/46] soup fix --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f48334b02..5488ca7c3 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -510,7 +510,7 @@ update_version() { echo "Updating the Security Onion version file." echo $NEWVERSION > /etc/soversion echo $HOTFIXVERSION > /etc/sohotfix - sed -i "/ soversion:/c\ soversion: $NEWVERSION" /opt/so/saltstack/local/pillar/global.sls + sed -i "/ soversion:/c\ soversion: $NEWVERSION" /opt/so/saltstack/local/pillar/global/soc_global.sls } upgrade_check() { From 07a590dda856eea4a7741dc60e866c8de6ff7d18 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 23 Jun 2023 17:17:59 -0400 Subject: [PATCH 41/46] fix agent extract error during install; simplify logic --- setup/so-functions | 21 ++++++++++----------- setup/so-verify | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d43469edb..e1f82b58a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -974,16 +974,16 @@ detect_os() { } download_elastic_agent_artifacts() { - - if [[ $is_iso ]]; then - logCmd "tar -xf /nsm/elastic-fleet/artifacts/beats/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" - else - logCmd "mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/" - retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" "" "" - retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5 --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" "" "" + agentArchive=/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz + agentMd5=/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5 + beatsDir=/nsm/elastic-fleet/artifacts/beats/elastic-agent + logCmd "mkdir -p $beatsDir" + if [[ ! -f "$agentArchive" ]]; then + retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz --output $agentArchive" "" "" + retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5 --output $agentMd5" "" "" - SOURCEHASH=$(md5sum /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz | awk '{ print $1 }') - HASH=$(cat /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5) + SOURCEHASH=$(md5sum $agentArchive | awk '{ print $1 }') + HASH=$(cat $agentMd5) if [[ "$HASH" == "$SOURCEHASH" ]]; then info "Elastic Agent source hash is good." @@ -991,9 +991,8 @@ download_elastic_agent_artifacts() { info "Unable to download the Elastic Agent source files." fail_setup fi - - logCmd "tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/" fi + logCmd "tar -xf $agentArchive -C $beatsDir" } installer_progress_loop() { diff --git a/setup/so-verify b/setup/so-verify index 7e010722c..918610732 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -36,7 +36,7 @@ log_has_errors() { # Failed to restart snapd.mounts-pre.target: Operation refused, unit snapd.mounts-pre.target # may be requested by dependency only (it is configured to refuse manual start/stop). - grep -E "FAILED|Failed|failed|ERROR|Result: False" "$setup_log" | \ + grep -E "FAILED|Failed|failed|ERROR|Result: False|Error is not recoverable" "$setup_log" | \ grep -vE "The Salt Master has cached the public key for this node" | \ grep -vE "Minion failed to authenticate with the master" | \ grep -vE "Failed to connect to ::1" | \ From 85a8da6331446faf13349bde026831a5914b8e40 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sat, 24 Jun 2023 16:58:36 -0400 Subject: [PATCH 42/46] Change policy name --- .../files/integrations-dynamic/grid-nodes/import-zeek-logs.json | 2 +- .../files/integrations-dynamic/grid-nodes/zeek-logs.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes/import-zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes/import-zeek-logs.json index feaebf60b..4c22f0446 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes/import-zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes/import-zeek-logs.json @@ -8,7 +8,7 @@ "name": "import-zeek-logs", "namespace": "so", "description": "Zeek Import logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes/zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes/zeek-logs.json index e2dd069ab..2cec88bf2 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes/zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes/zeek-logs.json @@ -9,7 +9,7 @@ "name": "zeek-logs", "namespace": "so", "description": "Zeek logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "inputs": { "logs-logfile": { "enabled": true, From 2247cafe5fecd7579b99f6e71d516b2e49daa771 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sat, 24 Jun 2023 17:13:28 -0400 Subject: [PATCH 43/46] Change policy name --- .../{grid-nodes => grid-nodes_general}/import-zeek-logs.json | 0 .../{grid-nodes => grid-nodes_general}/zeek-logs.json | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename salt/elasticfleet/files/integrations-dynamic/{grid-nodes => grid-nodes_general}/import-zeek-logs.json (100%) rename salt/elasticfleet/files/integrations-dynamic/{grid-nodes => grid-nodes_general}/zeek-logs.json (100%) diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes/import-zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json similarity index 100% rename from salt/elasticfleet/files/integrations-dynamic/grid-nodes/import-zeek-logs.json rename to salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes/zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json similarity index 100% rename from salt/elasticfleet/files/integrations-dynamic/grid-nodes/zeek-logs.json rename to salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json From 70fb28a8b3b053486229021e9309475fa965bb2e Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 26 Jun 2023 08:31:41 -0400 Subject: [PATCH 44/46] Update README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 72f2d34fe..3ab976bb5 100644 --- a/README.md +++ b/README.md @@ -5,22 +5,22 @@ Security Onion 2.4 Release Candidate 1 (RC1) is here! ## Screenshots Alerts -![Alerts](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/39_alerts.png) +![Alerts](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/50_alerts.png) Dashboards -![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/40_dashboards.png) +![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/51_dashboards.png) Hunt -![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/41_hunt.png) +![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/52_hunt.png) PCAP -![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/42_pcap.png) +![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_pcap.png) Grid -![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/46_grid.png) +![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_grid.png) Config -![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/50_config.png) +![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/61_config.png) ### Release Notes From 9530901d1dce514b12c321244633fe6c23eb4c7d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 26 Jun 2023 09:09:55 -0400 Subject: [PATCH 45/46] Remove local file check --- salt/manager/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 5488ca7c3..a4e22040d 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -845,8 +845,8 @@ main() { fi fi - echo "Checking for local modifications." - check_local_mods + #echo "Checking for local modifications." + #check_local_mods echo "Checking sudoers file." check_sudoers From 7aae72cfcff94fd5da98b5534b81c244c75d1573 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 26 Jun 2023 11:23:02 -0400 Subject: [PATCH 46/46] Update packages.sls --- salt/desktop/packages.sls | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/desktop/packages.sls b/salt/desktop/packages.sls index 9c3e8ccd2..4bf5122fb 100644 --- a/salt/desktop/packages.sls +++ b/salt/desktop/packages.sls @@ -43,6 +43,7 @@ desktop_packages: - bpftool - bzip2 - chkconfig + - chromium - chrony - cinnamon - cinnamon-control-center @@ -67,6 +68,7 @@ desktop_packages: - dosfstools - dracut-config-rescue - dracut-live + - dsniff - e2fsprogs - ed - efi-filesystem @@ -192,6 +194,7 @@ desktop_packages: - nemo-preview - net-tools - netronome-firmware + - ngrep - nm-connection-editor - nmap-ncat - nvme-cli @@ -220,6 +223,7 @@ desktop_packages: - psacct - pt-sans-fonts - python3-libselinux + - python3-scapy - qemu-guest-agent - quota - realmd @@ -251,6 +255,7 @@ desktop_packages: - smc-meera-fonts - sos - spice-vdagent + - ssldump - sssd - sssd-common - sssd-kcm @@ -263,6 +268,7 @@ desktop_packages: - systemd-udev - tar - tcpdump + - tcpflow - teamd - thai-scalable-waree-fonts - time @@ -282,8 +288,10 @@ desktop_packages: - vim-powerline - virt-what - wget + - whois - which - wireplumber + - wireshark - words - xdg-user-dirs-gtk - xed