From 310a6b4f27a02e8a748966ac345cccc2498da3d9 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Tue, 24 Oct 2023 14:21:01 -0400 Subject: [PATCH 01/78] Add kibana curl config --- salt/common/init.sls | 1 + salt/kibana/files/curl.config.template | 1 + salt/manager/init.sls | 1 + salt/manager/kibana.sls | 7 +++++++ 4 files changed, 10 insertions(+) create mode 100644 salt/kibana/files/curl.config.template create mode 100644 salt/manager/kibana.sls diff --git a/salt/common/init.sls b/salt/common/init.sls index f50f0c61b..8c0089fc0 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -8,6 +8,7 @@ include: - common.packages {% if GLOBALS.role in GLOBALS.manager_roles %} - manager.elasticsearch # needed for elastic_curl_config state + - manager.kibana {% endif %} net.core.wmem_default: diff --git a/salt/kibana/files/curl.config.template b/salt/kibana/files/curl.config.template new file mode 100644 index 000000000..7ac434017 --- /dev/null +++ b/salt/kibana/files/curl.config.template @@ -0,0 +1 @@ +user = "{{ salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:user', 'NO_USER_SET') }}:{{ salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:pass', 'NO_PW_SET') }}" diff --git a/salt/manager/init.sls b/salt/manager/init.sls index e808325ef..cbe3455fe 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -16,6 +16,7 @@ include: - kibana.secrets - manager.sync_es_users - manager.elasticsearch + - manager.kibana repo_log_dir: file.directory: diff --git a/salt/manager/kibana.sls b/salt/manager/kibana.sls new file mode 100644 index 000000000..eb2b968d6 --- /dev/null +++ b/salt/manager/kibana.sls @@ -0,0 +1,7 @@ +elastic_curl_config_distributed: + file.managed: + - name: /opt/so/conf/kibana/curl.config + - source: salt://kibana/files/curl.config.template + - template: jinja + - mode: 600 + - show_changes: False From 3ad480453a192c9fada8bf3aae3d6dd254029093 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Wed, 25 Oct 2023 07:20:07 -0400 Subject: [PATCH 02/78] Rename to remove dupe --- salt/manager/kibana.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/kibana.sls b/salt/manager/kibana.sls index eb2b968d6..f9aad3f05 100644 --- a/salt/manager/kibana.sls +++ b/salt/manager/kibana.sls @@ -1,4 +1,4 @@ -elastic_curl_config_distributed: +kibana_curl_config_distributed: file.managed: - name: /opt/so/conf/kibana/curl.config - source: salt://kibana/files/curl.config.template From a3e6b1ee1d495f8dae23175bf762378873d91507 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 25 Oct 2023 09:26:36 -0400 Subject: [PATCH 03/78] change generate_ssl wait_for_salt_minion --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index b64daaa92..fd5bc790f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1258,7 +1258,7 @@ generate_ssl() { # if the install type is a manager then we need to wait for the minion to be ready before trying # to run the ssl state since we need the minion to sign the certs if [[ "$install_type" =~ ^(EVAL|MANAGER|MANAGERSEARCH|STANDALONE|IMPORT|HELIXSENSOR)$ ]]; then - wait_for_salt_minion "$MINION_ID" "5" "$setup_log" || fail_setup + (wait_for_salt_minion "$MINION_ID" "5" '/dev/stdout' || fail_setup) 2>&1 | tee -a "$setup_log" fi info "Applying SSL state" logCmd "salt-call state.apply ssl -l info" From a887551dad8a9fed9a2b3f4d3b5e506d926a5e68 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 25 Oct 2023 15:22:47 -0400 Subject: [PATCH 04/78] Annotation changes for warm node --- salt/elasticsearch/soc_elasticsearch.yaml | 262 ++++++++++++++++------ 1 file changed, 197 insertions(+), 65 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index e4de29e00..f04ba08e0 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -33,7 +33,6 @@ elasticsearch: flood_stage: description: The max percentage of used disk space that will cause the node to take protective actions, such as blocking incoming events. helpLink: elasticsearch.html - script: max_compilations_rate: description: Max rate of script compilations permitted in the Elasticsearch cluster. Larger values will consume more resources. @@ -48,6 +47,11 @@ elasticsearch: helpLink: elasticsearch.html index_settings: global_overrides: + index_sorting: + description: Sorts the index by event time, at the cost of additional processing resource consumption. + global: True + advanced: True + helpLink: elasticsearch.html index_template: template: settings: @@ -57,32 +61,6 @@ elasticsearch: forcedType: int global: True helpLink: elasticsearch.html - so-logs: &indexSettings - index_sorting: - description: Sorts the index by event time, at the cost of additional processing resource consumption. - global: True - helpLink: elasticsearch.html - index_template: - index_patterns: - description: Patterns for matching multiple indices or tables. - forceType: "[]string" - multiline: True - global: True - helpLink: elasticsearch.html - template: - settings: - index: - number_of_replicas: - description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. - forcedType: int - global: True - helpLink: elasticsearch.html - mapping: - total_fields: - limit: - description: Max number of fields that can exist on a single index. Larger values will consume more resources. - global: True - helpLink: elasticsearch.html refresh_interval: description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation. global: True @@ -100,44 +78,7 @@ elasticsearch: description: The order to sort by. Must set index_sorting to True. global: True helpLink: elasticsearch.html - mappings: - _meta: - package: - name: - description: Meta settings for the mapping. - global: True - helpLink: elasticsearch.html - managed_by: - description: Meta settings for the mapping. - global: True - helpLink: elasticsearch.html - managed: - description: Meta settings for the mapping. - forcedType: bool - global: True - helpLink: elasticsearch.html - composed_of: - description: The index template is composed of these component templates. - forcedType: "[]string" - global: True - helpLink: elasticsearch.html - priority: - description: The priority of the index template. - forcedType: int - global: True - helpLink: elasticsearch.html - data_stream: - hidden: - description: Hide the data stream. - forcedType: bool - global: True - helpLink: elasticsearch.html - allow_custom_routing: - description: Allow custom routing for the data stream. - forcedType: bool - global: True - helpLink: elasticsearch.html - policy: + policy: phases: hot: min_age: @@ -160,6 +101,27 @@ elasticsearch: description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index. global: True helpLink: elasticsearch.html + warm: + min_age: + description: Minimum age of index. This determines when the index should be moved to the warm tier. + global: True + helpLink: elasticsearch.html + actions: + set_priority: + priority: + description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + forcedType: int + global: True + helpLink: elasticsearch.html + rollover: + max_age: + description: Maximum age of index. Once an index reaches this limit, it will be rolled over into a new index. + global: True + helpLink: elasticsearch.html + max_primary_shard_size: + description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index. + global: True + helpLink: elasticsearch.html cold: min_age: description: Minimum age of index. This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. @@ -177,20 +139,190 @@ elasticsearch: description: Minimum age of index. This determines when the index should be deleted. global: True helpLink: elasticsearch.html + so-logs: &indexSettings + index_sorting: + description: Sorts the index by event time, at the cost of additional processing resource consumption. + global: True + advanced: True + helpLink: elasticsearch.html + index_template: + index_patterns: + description: Patterns for matching multiple indices or tables. + forceType: "[]string" + multiline: True + global: True + advanced: True + helpLink: elasticsearch.html + template: + settings: + index: + number_of_replicas: + description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. + forcedType: int + global: True + advanced: True + helpLink: elasticsearch.html + mapping: + total_fields: + limit: + description: Max number of fields that can exist on a single index. Larger values will consume more resources. + global: True + advanced: True + helpLink: elasticsearch.html + refresh_interval: + description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation. + global: True + advanced: True + helpLink: elasticsearch.html + number_of_shards: + description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs. + global: True + advanced: True + helpLink: elasticsearch.html + sort: + field: + description: The field to sort by. Must set index_sorting to True. + global: True + advanced: True + helpLink: elasticsearch.html + order: + description: The order to sort by. Must set index_sorting to True. + global: True + advanced: True + helpLink: elasticsearch.html + mappings: + _meta: + package: + name: + description: Meta settings for the mapping. + global: True + advanced: True + helpLink: elasticsearch.html + managed_by: + description: Meta settings for the mapping. + global: True + advanced: True + helpLink: elasticsearch.html + managed: + description: Meta settings for the mapping. + forcedType: bool + global: True + advanced: True + helpLink: elasticsearch.html + composed_of: + description: The index template is composed of these component templates. + forcedType: "[]string" + global: True + advanced: True + helpLink: elasticsearch.html + priority: + description: The priority of the index template. + forcedType: int + global: True + advanced: True + helpLink: elasticsearch.html + data_stream: + hidden: + description: Hide the data stream. + forcedType: bool + global: True + advanced: True + helpLink: elasticsearch.html + allow_custom_routing: + description: Allow custom routing for the data stream. + forcedType: bool + global: True + advanced: True + helpLink: elasticsearch.html + policy: + phases: + hot: + min_age: + description: Minimum age of index. This determines when the index should be moved to the hot tier. + global: True + advanced: True + helpLink: elasticsearch.html + actions: + set_priority: + priority: + description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + forcedType: int + global: True + advanced: True + helpLink: elasticsearch.html + rollover: + max_age: + description: Maximum age of index. Once an index reaches this limit, it will be rolled over into a new index. + global: True + advanced: True + helpLink: elasticsearch.html + max_primary_shard_size: + description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index. + global: True + advanced: True + helpLink: elasticsearch.html + warm: + min_age: + description: Minimum age of index. This determines when the index should be moved to the hot tier. + global: True + advanced: True + helpLink: elasticsearch.html + actions: + set_priority: + priority: + description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + forcedType: int + global: True + advanced: True + helpLink: elasticsearch.html + rollover: + max_age: + description: Maximum age of index. Once an index reaches this limit, it will be rolled over into a new index. + global: True + advanced: True + helpLink: elasticsearch.html + max_primary_shard_size: + description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index. + global: True + advanced: True + helpLink: elasticsearch.html + cold: + min_age: + description: Minimum age of index. This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + global: True + advanced: True + helpLink: elasticsearch.html + actions: + set_priority: + priority: + description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + forcedType: int + global: True + advanced: True + helpLink: elasticsearch.html + delete: + min_age: + description: Minimum age of index. This determines when the index should be deleted. + global: True + advanced: True + helpLink: elasticsearch.html _meta: package: name: description: Meta settings for the mapping. global: True + advanced: True helpLink: elasticsearch.html managed_by: description: Meta settings for the mapping. global: True + advanced: True helpLink: elasticsearch.html managed: description: Meta settings for the mapping. forcedType: bool global: True + advanced: True helpLink: elasticsearch.html so-logs-system_x_auth: *indexSettings so-logs-system_x_syslog: *indexSettings From 6fb0c5dbfe5800c59a078a7aed80b0be1a8ed1c8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 25 Oct 2023 15:37:36 -0400 Subject: [PATCH 05/78] Annotation changes for warm node --- salt/elasticsearch/soc_elasticsearch.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index f04ba08e0..764de3c44 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -477,3 +477,18 @@ elasticsearch: so-strelka: *indexSettings so-syslog: *indexSettings so-zeek: *indexSettings + so_roles: + so-maanger: &soroleSettings + node: + roles: + description: List of Elasticsearch roles that the node should have. Blank assumes all roles + forcedType: "[]string" + global: False + advanced: True + helpLink: elasticsearch.html + so-managersearch: *soroleSettings + so-standalone: *soroleSettings + so-searchnode: *soroleSettings + so-heavynode: *soroleSettings + so-eval: *soroleSettings + so-import: *soroleSettings \ No newline at end of file From 1ae8896a05b6b07c4bdaf664f72ecc56544c4c4f Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 25 Oct 2023 15:47:40 -0400 Subject: [PATCH 06/78] Update config.map.jinja --- salt/elasticsearch/config.map.jinja | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 37447cabb..c98d96cc0 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -20,20 +20,12 @@ {% for NODE in ES_LOGSTASH_NODES %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %} {% endfor %} - {% if grains.id.split('_') | last == 'manager' %} - {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['master','data','remote_cluster_client','transform']}) %} - {% else %} - {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['master', 'data_hot', 'remote_cluster_client']}) %} - {% endif %} {% endif %} {% elif grains.id.split('_') | last == 'searchnode' %} - {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['data_hot', 'ingest']}) %} {% if HIGHLANDER %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.roles.extend(['ml', 'master', 'transform']) %} {% endif %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.manager]}}) %} -{% elif grains.id.split('_') | last == 'heavynode' %} - {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client', 'ingest']}) %} {% endif %} {% if HIGHLANDER %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.xpack.ml.update({'enabled': true}) %} @@ -53,3 +45,5 @@ {% endif %} {% endfor %} {% endif %} + +{% do ELASTICSEARCHMERGED.elasticsearch.config.node.update({'roles': ELASTICSEARCHMERGED.so_roles[GLOBALS.role].node.roles}) %} From af4b34801f958b9110e4de3837b01e3d8d341018 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 25 Oct 2023 15:48:27 -0400 Subject: [PATCH 07/78] Update defaults.yaml --- salt/elasticsearch/defaults.yaml | 38 ++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 1296ef549..5449df506 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -4920,3 +4920,41 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false + so_roles: + so-manager: + node: + roles: + - master + - data + - remote_cluster_client + - transform + so-managersearch: + node: + roles: + - master + - data_hot + - remote_cluster_client + so-standalone: + node: + roles: + - master + - data_hot + - remote_cluster_client + so-searchnode: + node: + roles: + - data_hot + - ingest + so-heavynode: + node: + roles: + - master + - data + - remote_cluster_client + - ingest + so-eval: + node: + roles: [] + so-import: + node: + roles: [] From dc53b49f15ed56cfa504193b1f22ee8adc1993c1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 25 Oct 2023 15:53:39 -0400 Subject: [PATCH 08/78] Update soup --- salt/manager/tools/sbin/soup | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index fc07765b8..0666e25ae 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -403,6 +403,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.4 ]] && up_to_2.4.5 [[ "$INSTALLEDVERSION" == 2.4.5 ]] && up_to_2.4.10 [[ "$INSTALLEDVERSION" == 2.4.10 ]] && up_to_2.4.20 + [[ "$INSTALLEDVERSION" == 2.4.20 ]] && up_to_2.4.30 true } @@ -414,7 +415,8 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.3 ]] && post_to_2.4.4 [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5 [[ "$POSTVERSION" == 2.4.5 ]] && post_to_2.4.10 - [[ "$POSTVERSION" == 2.4.10 ]] && post_to_2.4.20 + [[ "$POSTVERSION" == 2.4.10 ]] && post_to_2.4.20 + [[ "$POSTVERSION" == 2.4.20 ]] && post_to_2.4.30 true } @@ -446,6 +448,11 @@ post_to_2.4.20() { POSTVERSION=2.4.20 } +post_to_2.4.30() { + echo "Nothing to apply" + POSTVERSION=2.4.30 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -523,6 +530,12 @@ up_to_2.4.20() { INSTALLEDVERSION=2.4.20 } +up_to_2.4.30() { + echo "Nothing to do for 2.4.30" + + INSTALLEDVERSION=2.4.30 +} + determine_elastic_agent_upgrade() { if [[ $is_airgap -eq 0 ]]; then update_elastic_agent_airgap From 19fdc9319b900e315427af8364fca73d15289ac1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 25 Oct 2023 15:58:26 -0400 Subject: [PATCH 09/78] fix role update --- salt/elasticsearch/config.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index c98d96cc0..cd0cd8974 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -46,4 +46,4 @@ {% endfor %} {% endif %} -{% do ELASTICSEARCHMERGED.elasticsearch.config.node.update({'roles': ELASTICSEARCHMERGED.so_roles[GLOBALS.role].node.roles}) %} +{% do ELASTICSEARCHMERGED.config.node.update({'roles': ELASTICSEARCHMERGED.so_roles[GLOBALS.role].node.roles}) %} From d1170cb69f1f197b638151ffef9561d50b3122f9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 25 Oct 2023 16:05:20 -0400 Subject: [PATCH 10/78] Update soc_elasticsearch.yaml --- salt/elasticsearch/soc_elasticsearch.yaml | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 764de3c44..fde8d234f 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -101,27 +101,6 @@ elasticsearch: description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index. global: True helpLink: elasticsearch.html - warm: - min_age: - description: Minimum age of index. This determines when the index should be moved to the warm tier. - global: True - helpLink: elasticsearch.html - actions: - set_priority: - priority: - description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. - forcedType: int - global: True - helpLink: elasticsearch.html - rollover: - max_age: - description: Maximum age of index. Once an index reaches this limit, it will be rolled over into a new index. - global: True - helpLink: elasticsearch.html - max_primary_shard_size: - description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index. - global: True - helpLink: elasticsearch.html cold: min_age: description: Minimum age of index. This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. @@ -491,4 +470,4 @@ elasticsearch: so-searchnode: *soroleSettings so-heavynode: *soroleSettings so-eval: *soroleSettings - so-import: *soroleSettings \ No newline at end of file + so-import: *soroleSettings From 5f168a33edaa8a4c0801340d8f8eaf23d68279ff Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 25 Oct 2023 16:16:01 -0400 Subject: [PATCH 11/78] Update defaults.yaml --- salt/elasticsearch/defaults.yaml | 67 ++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 5449df506..807ca9ea9 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -4922,39 +4922,46 @@ elasticsearch: allow_custom_routing: false so_roles: so-manager: - node: - roles: - - master - - data - - remote_cluster_client - - transform + config: + node: + roles: + - master + - data + - remote_cluster_client + - transform so-managersearch: - node: - roles: - - master - - data_hot - - remote_cluster_client + config: + node: + roles: + - master + - data_hot + - remote_cluster_client so-standalone: - node: - roles: - - master - - data_hot - - remote_cluster_client + config: + node: + roles: + - master + - data_hot + - remote_cluster_client so-searchnode: - node: - roles: - - data_hot - - ingest + config: + node: + roles: + - data_hot + - ingest so-heavynode: - node: - roles: - - master - - data - - remote_cluster_client - - ingest + config: + node: + roles: + - master + - data + - remote_cluster_client + - ingest so-eval: - node: - roles: [] + config: + node: + roles: [] so-import: - node: - roles: [] + config: + node: + roles: [] From 807b40019fc098ea8c0c8d70355e1086dd3dbebc Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 25 Oct 2023 16:16:48 -0400 Subject: [PATCH 12/78] Update soc_elasticsearch.yaml --- salt/elasticsearch/soc_elasticsearch.yaml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index fde8d234f..d456dcbfc 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -458,13 +458,14 @@ elasticsearch: so-zeek: *indexSettings so_roles: so-maanger: &soroleSettings - node: - roles: - description: List of Elasticsearch roles that the node should have. Blank assumes all roles - forcedType: "[]string" - global: False - advanced: True - helpLink: elasticsearch.html + config: + node: + roles: + description: List of Elasticsearch roles that the node should have. Blank assumes all roles + forcedType: "[]string" + global: False + advanced: True + helpLink: elasticsearch.html so-managersearch: *soroleSettings so-standalone: *soroleSettings so-searchnode: *soroleSettings From 39abe19cfd9f6ad0553ade33614ae2cee2829a8d Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 25 Oct 2023 16:17:06 -0400 Subject: [PATCH 13/78] Update config.map.jinja --- salt/elasticsearch/config.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index cd0cd8974..4e57199af 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -46,4 +46,4 @@ {% endfor %} {% endif %} -{% do ELASTICSEARCHMERGED.config.node.update({'roles': ELASTICSEARCHMERGED.so_roles[GLOBALS.role].node.roles}) %} +{% do ELASTICSEARCHMERGED.config.node.update({'roles': ELASTICSEARCHMERGED.so_roles[GLOBALS.role].config.node.roles}) %} From 88fb7d06e673854a2c8bf22a6c2942493cec77a1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 25 Oct 2023 16:20:28 -0400 Subject: [PATCH 14/78] Annotation changes for warm node --- salt/elasticsearch/soc_elasticsearch.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index d456dcbfc..368f15196 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -47,11 +47,6 @@ elasticsearch: helpLink: elasticsearch.html index_settings: global_overrides: - index_sorting: - description: Sorts the index by event time, at the cost of additional processing resource consumption. - global: True - advanced: True - helpLink: elasticsearch.html index_template: template: settings: From 6d6292714f5053b76b5aff8e6bbea37341772d1e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 25 Oct 2023 16:21:47 -0400 Subject: [PATCH 15/78] Annotation changes for warm node --- salt/elasticsearch/soc_elasticsearch.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 368f15196..46306203a 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -73,7 +73,6 @@ elasticsearch: description: The order to sort by. Must set index_sorting to True. global: True helpLink: elasticsearch.html - policy: phases: hot: min_age: From 01810a782cde1f49fcd86c153c81e99b5df14ce3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 25 Oct 2023 16:46:30 -0400 Subject: [PATCH 16/78] Annotation changes for warm node --- salt/elasticsearch/soc_elasticsearch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 46306203a..e3d257f11 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -451,7 +451,7 @@ elasticsearch: so-syslog: *indexSettings so-zeek: *indexSettings so_roles: - so-maanger: &soroleSettings + so-manger: &soroleSettings config: node: roles: From 891ea997e75fdbecc4a043ba2f3a41be51439cdc Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Oct 2023 12:25:37 +0000 Subject: [PATCH 17/78] Add lifecycle policies and warm settings --- salt/elasticsearch/defaults.yaml | 13704 +++++++++++++++++++---------- 1 file changed, 8973 insertions(+), 4731 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 807ca9ea9..9aef09876 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,55 +1,16 @@ elasticsearch: - enabled: False - retention: - retention_pct: 50 config: - node: {} + action: + destructive_requires_name: true cluster: routing: allocation: disk: threshold_enabled: true watermark: - low: 80% - high: 85% flood_stage: 90% - network: - host: 0.0.0.0 - path: - logs: /var/log/elasticsearch - action: - destructive_requires_name: true - transport: - bind_host: 0.0.0.0 - publish_port: 9300 - xpack: - ml: - enabled: false - security: - enabled: true - authc: - anonymous: - authz_exception: true - roles: [] - username: _anonymous - transport: - ssl: - enabled: true - verification_mode: none - key: /usr/share/elasticsearch/config/elasticsearch.key - certificate: /usr/share/elasticsearch/config/elasticsearch.crt - certificate_authorities: - - /usr/share/elasticsearch/config/ca.crt - http: - ssl: - enabled: true - client_authentication: none - key: /usr/share/elasticsearch/config/elasticsearch.key - certificate: /usr/share/elasticsearch/config/elasticsearch.crt - certificate_authorities: - - /usr/share/elasticsearch/config/ca.crt - script: - max_compilations_rate: 20000/1m + high: 85% + low: 80% indices: id_field_data: enabled: false @@ -57,3833 +18,8553 @@ elasticsearch: org: elasticsearch: deprecation: ERROR + network: + host: 0.0.0.0 + node: {} + path: + logs: /var/log/elasticsearch + script: + max_compilations_rate: 20000/1m + transport: + bind_host: 0.0.0.0 + publish_port: 9300 + xpack: + ml: + enabled: false + security: + authc: + anonymous: + authz_exception: true + roles: [] + username: _anonymous + enabled: true + http: + ssl: + certificate: /usr/share/elasticsearch/config/elasticsearch.crt + certificate_authorities: + - /usr/share/elasticsearch/config/ca.crt + client_authentication: none + enabled: true + key: /usr/share/elasticsearch/config/elasticsearch.key + transport: + ssl: + certificate: /usr/share/elasticsearch/config/elasticsearch.crt + certificate_authorities: + - /usr/share/elasticsearch/config/ca.crt + enabled: true + key: /usr/share/elasticsearch/config/elasticsearch.key + verification_mode: none + enabled: false index_settings: global_overrides: index_template: template: settings: index: + lifecycle: + name: global_overrides-logs number_of_replicas: default_placeholder - so-logs: - index_sorting: False - index_template: - index_patterns: - - "logs-*-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5001 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - composed_of: - - "so-data-streams-mappings" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - - "so-logs-mappings" - - "so-logs-settings" - priority: 225 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb cold: - min_age: 30d actions: set_priority: priority: 0 + min_age: 30d delete: - min_age: 365d actions: delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-system_x_auth: - index_sorting: False - index_template: - index_patterns: - - "logs-system.auth*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "event-mappings" - - "logs-system.auth@package" - - "logs-system.auth@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-system_x_syslog: - index_sorting: False - index_template: - index_patterns: - - "logs-system.syslog*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "event-mappings" - - "logs-system.syslog@package" - - "logs-system.syslog@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-system_x_system: - index_sorting: False - index_template: - index_patterns: - - "logs-system.system*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "event-mappings" - - "logs-system.system@package" - - "logs-system.system@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-system_x_application: - index_sorting: False - index_template: - index_patterns: - - "logs-system.application*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "event-mappings" - - "logs-system.application@package" - - "logs-system.application@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-system_x_security: - index_sorting: False - index_template: - index_patterns: - - "logs-system.security*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "event-mappings" - - "logs-system.security@package" - - "logs-system.security@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-windows_x_forwarded: - index_sorting: False - index_template: - index_patterns: - - "logs-windows.forwarded*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-windows.forwarded@package" - - "logs-windows.forwarded@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-windows_x_powershell: - index_sorting: False - index_template: - index_patterns: - - "logs-windows.powershell-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-windows.powershell@package" - - "logs-windows.powershell@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-windows_x_powershell_operational: - index_sorting: False - index_template: - index_patterns: - - "logs-windows.powershell_operational-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-windows.powershell_operational@package" - - "logs-windows.powershell_operational@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-windows_x_sysmon_operational: - index_sorting: False - index_template: - index_patterns: - - "logs-windows.sysmon_operational-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-windows.sysmon_operational@package" - - "logs-windows.sysmon_operational@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-apache_x_access: - index_sorting: False - index_template: - index_patterns: - - "logs-apache.access-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-apache.access@package" - - "logs-apache.access@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-apache_x_error: - index_sorting: False - index_template: - index_patterns: - - "logs-apache.error-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-apache.error@package" - - "logs-apache.error@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-auditd_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-auditd.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-auditd.log@package" - - "logs-auditd.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-aws_x_cloudtrail: - index_sorting: False - index_template: - index_patterns: - - "logs-aws.cloudtrail-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-aws.cloudtrail@package" - - "logs-aws.cloudtrail@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-aws_x_cloudwatch_logs: - index_sorting: False - index_template: - index_patterns: - - "logs-aws.cloudwatch_logs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-aws.cloudwatch_logs@package" - - "logs-aws.cloudwatch_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-aws_x_ec2_logs: - index_sorting: False - index_template: - index_patterns: - - "logs-aws.ec2_logs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-aws.ec2_logs@package" - - "logs-aws.ec2_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-aws_x_elb_logs: - index_sorting: False - index_template: - index_patterns: - - "logs-aws.elb_logs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-aws.elb_logs@package" - - "logs-aws.elb_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-aws_x_firewall_logs: - index_sorting: False - index_template: - index_patterns: - - "logs-aws.firewall_logs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-aws.firewall_logs@package" - - "logs-aws.firewall_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-aws_x_route53_public_logs: - index_sorting: False - index_template: - index_patterns: - - "logs-aws.route53_public_logs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-aws.route53_public_logs@package" - - "logs-aws.route53_public_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-aws_x_route53_resolver_logs: - index_sorting: False - index_template: - index_patterns: - - "logs-aws.route53_resolver_logs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-aws.route53_resolver_logs@package" - - "logs-aws.route53_resolver_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-aws_x_s3access: - index_sorting: False - index_template: - index_patterns: - - "logs-aws.s3access-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-aws.s3access@package" - - "logs-aws.s3access@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-aws_x_vpcflow: - index_sorting: False - index_template: - index_patterns: - - "logs-aws.vpcflow-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-aws.vpcflow@package" - - "logs-aws.vpcflow@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-aws_x_waf: - index_sorting: False - index_template: - index_patterns: - - "logs-aws.waf-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-aws.waf@package" - - "logs-aws.waf@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-azure_x_activitylogs: - index_sorting: False - index_template: - index_patterns: - - "logs-azure.activitylogs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-azure.activitylogs@package" - - "logs-azure.activitylogs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-azure_x_application_gateway: - index_sorting: False - index_template: - index_patterns: - - "logs-azure.application_gateway-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-azure.application_gateway@package" - - "logs-azure.application_gateway@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-azure_x_auditlogs: - index_sorting: False - index_template: - index_patterns: - - "logs-azure.auditlogs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-azure.auditlogs@package" - - "logs-azure.auditlogs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-azure_x_eventhub: - index_sorting: False - index_template: - index_patterns: - - "logs-azure.eventhub-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-azure.eventhub@package" - - "logs-azure.eventhub@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-azure_x_firewall_logs: - index_sorting: False - index_template: - index_patterns: - - "logs-azure.firewall_logs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-azure.firewall_logs@package" - - "logs-azure.firewall_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-azure_x_identity_protection: - index_sorting: False - index_template: - index_patterns: - - "logs-azure.identity_protection-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-azure.identity_protection@package" - - "logs-azure.identity_protection@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-azure_x_platformlogs: - index_sorting: False - index_template: - index_patterns: - - "logs-azure.platformlogs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-azure.platformlogs@package" - - "logs-azure.platformlogs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-azure_x_provisioning: - index_sorting: False - index_template: - index_patterns: - - "logs-azure.provisioning-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-azure.provisioning@package" - - "logs-azure.provisioning@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-azure_x_signinlogs: - index_sorting: False - index_template: - index_patterns: - - "logs-azure.signinlogs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-azure.signinlogs@package" - - "logs-azure.signinlogs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-azure_x_springcloudlogs: - index_sorting: False - index_template: - index_patterns: - - "logs-azure.springcloudlogs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-azure.springcloudlogs@package" - - "logs-azure.springcloudlogs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-barracuda_x_waf: - index_sorting: False - index_template: - index_patterns: - - "logs-barracuda.waf-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-barracuda.waf@package" - - "logs-barracuda.waf@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-cisco_asa_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-cisco_asa.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-cisco_asa.log@package" - - "logs-cisco_asa.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-cloudflare_x_audit: - index_sorting: False - index_template: - index_patterns: - - "logs-cloudflare.audit-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-cloudflare.audit@package" - - "logs-cloudflare.audit@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-cloudflare_x_logpull: - index_sorting: False - index_template: - index_patterns: - - "logs-cloudflare.logpull-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-cloudflare.logpull@package" - - "logs-cloudflare.logpull@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-crowdstrike_x_falcon: - index_sorting: False - index_template: - index_patterns: - - "logs-crowdstrike.falcon-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-crowdstrike.falcon@package" - - "logs-crowdstrike.falcon@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-crowdstrike_x_fdr: - index_sorting: False - index_template: - index_patterns: - - "logs-crowdstrike.fdr-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-crowdstrike.fdr@package" - - "logs-crowdstrike.fdr@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-darktrace_x_ai_analyst_alert: - index_sorting: False - index_template: - index_patterns: - - "logs-darktrace.ai_analyst_alert-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-darktrace.ai_analyst_alert@package" - - "logs-darktrace.ai_analyst_alert@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-darktrace_x_model_breach_alert: - index_sorting: False - index_template: - index_patterns: - - "logs-darktrace.model_breach_alert-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-darktrace.model_breach_alert@package" - - "logs-darktrace.model_breach_alert@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-darktrace_x_system_status_alert: - index_sorting: False - index_template: - index_patterns: - - "logs-darktrace.system_status_alert-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-darktrace.system_status_alert@package" - - "logs-darktrace.system_status_alert@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-f5_bigip_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-f5_bigip.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-f5_bigip.log@package" - - "logs-f5_bigip.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-fim_x_event: - index_sorting: False - index_template: - index_patterns: - - "logs-fim.event-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-fim.event@package" - - "logs-fim.event@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-fortinet_x_clientendpoint: - index_sorting: False - index_template: - index_patterns: - - "logs-fortinet.clientendpoint-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-fortinet.clientendpoint@package" - - "logs-fortinet.clientendpoint@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-fortinet_x_firewall: - index_sorting: False - index_template: - index_patterns: - - "logs-fortinet.firewall-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-fortinet.firewall@package" - - "logs-fortinet.firewall@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-fortinet_x_fortimail: - index_sorting: False - index_template: - index_patterns: - - "logs-fortinet.fortimail-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-fortinet.fortimail@package" - - "logs-fortinet.fortimail@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-fortinet_x_fortimanager: - index_sorting: False - index_template: - index_patterns: - - "logs-fortinet.fortimanager-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-fortinet.fortimanager@package" - - "logs-fortinet.fortimanager@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-fortinet_fortigate_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-fortinet_fortigate.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-fortinet_fortigate.log@package" - - "logs-fortinet_fortigate.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-gcp_x_audit: - index_sorting: False - index_template: - index_patterns: - - "logs-gcp.audit-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-gcp.audit@package" - - "logs-gcp.audit@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-gcp_x_dns: - index_sorting: False - index_template: - index_patterns: - - "logs-gcp.dns-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-gcp.dns@package" - - "logs-gcp.dns@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-gcp_x_firewall: - index_sorting: False - index_template: - index_patterns: - - "logs-gcp.firewall-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-gcp.firewall@package" - - "logs-gcp.firewall@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-gcp_x_loadbalancing_logs: - index_sorting: False - index_template: - index_patterns: - - "logs-gcp.loadbalancing_logs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-gcp.loadbalancing_logs@package" - - "logs-gcp.loadbalancing_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-gcp_x_vpcflow: - index_sorting: False - index_template: - index_patterns: - - "logs-gcp.vpcflow-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-gcp.vpcflow@package" - - "logs-gcp.vpcflow@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-github_x_audit: - index_sorting: False - index_template: - index_patterns: - - "logs-github.audit-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-github.audit@package" - - "logs-github.audit@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-github_x_code_scanning: - index_sorting: False - index_template: - index_patterns: - - "logs-github.code_scanning-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-github.code_scanning@package" - - "logs-github.code_scanning@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-github_x_dependabot: - index_sorting: False - index_template: - index_patterns: - - "logs-github.dependabot-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-github.dependabot@package" - - "logs-github.dependabot@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-github_x_issues: - index_sorting: False - index_template: - index_patterns: - - "logs-github.issues-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-github.issues@package" - - "logs-github.issues@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-github_x_secret_scanning: - index_sorting: False - index_template: - index_patterns: - - "logs-github.secret_scanning-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-github.secret_scanning@package" - - "logs-github.secret_scanning@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-google_workspace_x_access_transparency: - index_sorting: False - index_template: - index_patterns: - - "logs-google_workspace.access_transparency-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-google_workspace.access_transparency@package" - - "logs-google_workspace.access_transparency@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-google_workspace_x_admin: - index_sorting: False - index_template: - index_patterns: - - "logs-google_workspace.admin-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-google_workspace.admin@package" - - "logs-google_workspace.admin@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-google_workspace_x_alert: - index_sorting: False - index_template: - index_patterns: - - "logs-google_workspace.alert-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-google_workspace.alert@package" - - "logs-google_workspace.alert@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-google_workspace_x_context_aware_access: - index_sorting: False - index_template: - index_patterns: - - "logs-google_workspace.context_aware_access-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-google_workspace.context_aware_access@package" - - "logs-google_workspace.context_aware_access@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-google_workspace_x_device: - index_sorting: False - index_template: - index_patterns: - - "logs-google_workspace.device-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-google_workspace.device@package" - - "logs-google_workspace.device@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-google_workspace_x_drive: - index_sorting: False - index_template: - index_patterns: - - "logs-google_workspace.drive-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-google_workspace.drive@package" - - "logs-google_workspace.drive@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-google_workspace_x_gcp: - index_sorting: False - index_template: - index_patterns: - - "logs-google_workspace.gcp-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-google_workspace.gcp@package" - - "logs-google_workspace.gcp@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-google_workspace_x_group_enterprise: - index_sorting: False - index_template: - index_patterns: - - "logs-google_workspace.group_enterprise-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-google_workspace.group_enterprise@package" - - "logs-google_workspace.group_enterprise@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-google_workspace_x_groups: - index_sorting: False - index_template: - index_patterns: - - "logs-google_workspace.groups-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-google_workspace.groups@package" - - "logs-google_workspace.groups@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-google_workspace_x_login: - index_sorting: False - index_template: - index_patterns: - - "logs-google_workspace.login-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-google_workspace.login@package" - - "logs-google_workspace.login@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-google_workspace_x_rules: - index_sorting: False - index_template: - index_patterns: - - "logs-google_workspace.rules-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-google_workspace.rules@package" - - "logs-google_workspace.rules@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-google_workspace_x_saml: - index_sorting: False - index_template: - index_patterns: - - "logs-google_workspace.saml-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-google_workspace.saml@package" - - "logs-google_workspace.saml@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-google_workspace_x_token: - index_sorting: False - index_template: - index_patterns: - - "logs-google_workspace.token-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-google_workspace.token@package" - - "logs-google_workspace.token@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-google_workspace_x_user_accounts: - index_sorting: False - index_template: - index_patterns: - - "logs-google_workspace.user_accounts-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-google_workspace.user_accounts@package" - - "logs-google_workspace.user_accounts@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-http_endpoint_x_generic: - index_sorting: False - index_template: - index_patterns: - - "logs-http_endpoint.generic-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-http_endpoint.generic@package" - - "logs-http_endpoint.generic@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-httpjson_x_generic: - index_sorting: False - index_template: - index_patterns: - - "logs-httpjson.generic-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-httpjson.generic@package" - - "logs-httpjson.generic@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-juniper_x_junos: - index_sorting: False - index_template: - index_patterns: - - "logs-juniper.junos-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-juniper.junos@package" - - "logs-juniper.junos@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-juniper_x_netscreen: - index_sorting: False - index_template: - index_patterns: - - "logs-juniper.netscreen-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-juniper.netscreen@package" - - "logs-juniper.netscreen@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-juniper_x_srx: - index_sorting: False - index_template: - index_patterns: - - "logs-juniper.srx-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-juniper.srx@package" - - "logs-juniper.srx@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-juniper_srx_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-juniper_srx.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-juniper_srx.log@package" - - "logs-juniper_srx.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-kafka_log_x_generic: - index_sorting: False - index_template: - index_patterns: - - "logs-kafka_log.generic-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-kafka_log.generic@package" - - "logs-kafka_log.generic@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-lastpass_x_detailed_shared_folder: - index_sorting: False - index_template: - index_patterns: - - "logs-lastpass.detailed_shared_folder-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-lastpass.detailed_shared_folder@package" - - "logs-lastpass.detailed_shared_folder@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-lastpass_x_event_report: - index_sorting: False - index_template: - index_patterns: - - "logs-lastpass.event_report-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-lastpass.event_report@package" - - "logs-lastpass.event_report@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-lastpass_x_user: - index_sorting: False - index_template: - index_patterns: - - "logs-lastpass.user-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-lastpass.user@package" - - "logs-lastpass.user@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-m365_defender_x_event: - index_sorting: False - index_template: - index_patterns: - - "logs-m365_defender.event-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-m365_defender.event@package" - - "logs-m365_defender.event@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-m365_defender_x_incident: - index_sorting: False - index_template: - index_patterns: - - "logs-m365_defender.incident-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-m365_defender.incident@package" - - "logs-m365_defender.incident@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-m365_defender_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-m365_defender.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-m365_defender.log@package" - - "logs-m365_defender.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-microsoft_defender_endpoint_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-microsoft_defender_endpoint.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-microsoft_defender_endpoint.log@package" - - "logs-microsoft_defender_endpoint.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-microsoft_dhcp_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-microsoft_dhcp.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-microsoft_dhcp.log@package" - - "logs-microsoft_dhcp.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-netflow_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-netflow.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-netflow.log@package" - - "logs-netflow.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-o365_x_audit: - index_sorting: False - index_template: - index_patterns: - - "logs-o365.audit-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-o365.audit@package" - - "logs-o365.audit@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-okta_x_system: - index_sorting: False - index_template: - index_patterns: - - "logs-okta.system-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-okta.system@package" - - "logs-okta.system@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-panw_x_panos: - index_sorting: False - index_template: - index_patterns: - - "logs-panw.panos-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-panw.panos@package" - - "logs-panw.panos@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-pfsense_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-pfsense.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-pfsense.log@package" - - "logs-pfsense.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-sentinel_one_x_activity: - index_sorting: False - index_template: - index_patterns: - - "logs-sentinel_one.activity-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-sentinel_one.activity@package" - - "logs-sentinel_one.activity@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-sentinel_one_x_agent: - index_sorting: False - index_template: - index_patterns: - - "logs-sentinel_one.agent-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-sentinel_one.agent@package" - - "logs-sentinel_one.agent@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-sentinel_one_x_alert: - index_sorting: False - index_template: - index_patterns: - - "logs-sentinel_one.alert-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-sentinel_one.alert@package" - - "logs-sentinel_one.alert@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-sentinel_one_x_group: - index_sorting: False - index_template: - index_patterns: - - "logs-sentinel_one.group-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-sentinel_one.group@package" - - "logs-sentinel_one.group@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-sentinel_one_x_threat: - index_sorting: False - index_template: - index_patterns: - - "logs-sentinel_one.threat-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-sentinel_one.threat@package" - - "logs-sentinel_one.threat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-sonicwall_firewall_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-sonicwall_firewall.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-sonicwall_firewall.log@package" - - "logs-sonicwall_firewall.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-symantec_endpoint_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-symantec_endpoint.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-symantec_endpoint.log@package" - - "logs-symantec_endpoint.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-ti_abusech_x_malware: - index_sorting: False - index_template: - index_patterns: - - "logs-ti_abusech.malware-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-ti_abusech.malware@package" - - "logs-ti_abusech.malware@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-ti_abusech_x_malwarebazaar: - index_sorting: False - index_template: - index_patterns: - - "logs-ti_abusech.malwarebazaar-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-ti_abusech.malwarebazaar@package" - - "logs-ti_abusech.malwarebazaar@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-ti_abusech_x_threatfox: - index_sorting: False - index_template: - index_patterns: - - "logs-ti_abusech.threatfox-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-ti_abusech.threatfox@package" - - "logs-ti_abusech.threatfox@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-ti_abusech_x_url: - index_sorting: False - index_template: - index_patterns: - - "logs-ti_abusech.url-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-ti_abusech.url@package" - - "logs-ti_abusech.url@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-ti_misp_x_threat: - index_sorting: False - index_template: - index_patterns: - - "logs-ti_misp.threat-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-ti_misp.threat@package" - - "logs-ti_misp.threat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-ti_misp_x_threat_attributes: - index_sorting: False - index_template: - index_patterns: - - "logs-ti_misp.threat_attributes-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-ti_misp.threat_attributes@package" - - "logs-ti_misp.threat_attributes@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-ti_otx_x_threat: - index_sorting: False - index_template: - index_patterns: - - "logs-ti_otx.threat-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-ti_otx.threat@package" - - "logs-ti_otx.threat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-ti_recordedfuture_x_latest_ioc-template: - index_sorting: False - index_template: - index_patterns: - - "logs-ti_recordedfuture.latest_ioc-template-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-ti_recordedfuture.latest_ioc-template@package" - - "logs-ti_recordedfuture.latest_ioc-template@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-ti_recordedfuture_x_threat: - index_sorting: False - index_template: - index_patterns: - - "logs-ti_recordedfuture.threat-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-ti_recordedfuture.threat@package" - - "logs-ti_recordedfuture.threat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-zscaler_zia_x_alerts: - index_sorting: False - index_template: - index_patterns: - - "logs-zscaler_zia.alerts-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-zscaler_zia.alerts@package" - - "logs-zscaler_zia.alerts@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-zscaler_zia_x_dns: - index_sorting: False - index_template: - index_patterns: - - "logs-zscaler_zia.dns-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-zscaler_zia.dns@package" - - "logs-zscaler_zia.dns@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-zscaler_zia_x_firewall: - index_sorting: False - index_template: - index_patterns: - - "logs-zscaler_zia.firewall-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-zscaler_zia.firewall@package" - - "logs-zscaler_zia.firewall@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-zscaler_zia_x_tunnel: - index_sorting: False - index_template: - index_patterns: - - "logs-zscaler_zia.tunnel-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-zscaler_zia.tunnel@package" - - "logs-zscaler_zia.tunnel@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-zscaler_zia_x_web: - index_sorting: False - index_template: - index_patterns: - - "logs-zscaler_zia.web-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-zscaler_zia.web@package" - - "logs-zscaler_zia.web@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-zscaler_zpa_x_app_connector_status: - index_sorting: False - index_template: - index_patterns: - - "logs-zscaler_zpa.app_connector_status-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-zscaler_zpa.app_connector_status@package" - - "logs-zscaler_zpa.app_connector_status@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-zscaler_zpa_x_audit: - index_sorting: False - index_template: - index_patterns: - - "logs-zscaler_zpa.audit-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-zscaler_zpa.audit@package" - - "logs-zscaler_zpa.audit@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-zscaler_zpa_x_browser_access: - index_sorting: False - index_template: - index_patterns: - - "logs-zscaler_zpa.browser_access-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-zscaler_zpa.browser_access@package" - - "logs-zscaler_zpa.browser_access@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-zscaler_zpa_x_user_activity: - index_sorting: False - index_template: - index_patterns: - - "logs-zscaler_zpa.user_activity-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-zscaler_zpa.user_activity@package" - - "logs-zscaler_zpa.user_activity@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-zscaler_zpa_x_user_status: - index_sorting: False - index_template: - index_patterns: - - "logs-zscaler_zpa.user_status-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-zscaler_zpa.user_status@package" - - "logs-zscaler_zpa.user_status@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-1password_x_item_usages: - index_sorting: False - index_template: - index_patterns: - - "logs-1password.item_usages-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-1password.item_usages@package" - - "logs-1password.item_usages@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-1password_x_signin_attempts: - index_sorting: False - index_template: - index_patterns: - - "logs-1password.signin_attempts-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-1password.signin_attempts@package" - - "logs-1password.signin_attempts@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-osquery-manager-actions: - index_sorting: False - index_template: - index_patterns: - - ".logs-osquery_manager.actions*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-osquery_manager.actions" - priority: 501 - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-osquery-manager-action_x_responses: - index_sorting: False - index_template: - index_patterns: - - ".logs-osquery_manager.action.responses*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-osquery_manager.action.responses" - priority: 501 - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-elastic_agent_x_apm_server: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.apm_server-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - composed_of: - - "logs-elastic_agent.apm_server@package" - - "logs-elastic_agent.apm_server@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: + min_age: 365d hot: - min_age: 0ms actions: - set_priority: - priority: 100 rollover: max_age: 30d max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-elastic_agent_x_auditbeat: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.auditbeat-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - composed_of: - - "logs-elastic_agent.auditbeat@package" - - "logs-elastic_agent.auditbeat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: set_priority: priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-elastic_agent_x_cloudbeat: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.cloudbeat-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - composed_of: - - "logs-elastic_agent.cloudbeat@package" - - "logs-elastic_agent.cloudbeat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - policy: - phases: - hot: min_age: 0ms + warm: actions: set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: + priority: 50 min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-elastic_agent_x_endpoint_security: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.endpoint_security-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - composed_of: - - "event-mappings" - - "logs-elastic_agent.endpoint_security@package" - - "logs-elastic_agent.endpoint_security@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-endpoint_x_alerts: - index_sorting: False - index_template: - index_patterns: - - "logs-endpoint.alerts-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - composed_of: - - "event-mappings" - - "logs-endpoint.alerts@custom" - - "logs-endpoint.alerts@package" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-endpoint_x_events_x_api: - index_sorting: False - index_template: - index_patterns: - - "logs-endpoint.events.api-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - composed_of: - - "event-mappings" - - "logs-endpoint.events.api@custom" - - "logs-endpoint.events.api@package" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-endpoint_x_events_x_file: - index_sorting: False - index_template: - index_patterns: - - "logs-endpoint.events.file-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - composed_of: - - "event-mappings" - - "logs-endpoint.events.file@custom" - - "logs-endpoint.events.file@package" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-endpoint_x_events_x_library: - index_sorting: False - index_template: - index_patterns: - - "logs-endpoint.events.library-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - composed_of: - - "event-mappings" - - "logs-endpoint.events.library@custom" - - "logs-endpoint.events.library@package" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-endpoint_x_events_x_network: - index_sorting: False - index_template: - index_patterns: - - "logs-endpoint.events.network-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - composed_of: - - "event-mappings" - - "logs-endpoint.events.network@custom" - - "logs-endpoint.events.network@package" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-endpoint_x_events_x_process: - index_sorting: False - index_template: - index_patterns: - - "logs-endpoint.events.process-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - composed_of: - - "event-mappings" - - "logs-endpoint.events.process@custom" - - "logs-endpoint.events.process@package" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-endpoint_x_events_x_registry: - index_sorting: False - index_template: - index_patterns: - - "logs-endpoint.events.registry-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - composed_of: - - "event-mappings" - - "logs-endpoint.events.registry@custom" - - "logs-endpoint.events.registry@package" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-endpoint_x_events_x_security: - index_sorting: False - index_template: - index_patterns: - - "logs-endpoint.events.security-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - composed_of: - - "event-mappings" - - "logs-endpoint.events.security@custom" - - "logs-endpoint.events.security@package" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-elastic_agent_x_filebeat: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.filebeat-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - composed_of: - - "event-mappings" - - "logs-elastic_agent.filebeat@package" - - "logs-elastic_agent.filebeat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-elastic_agent_x_fleet_server: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.fleet_server-*" - template: - settings: - index: - number_of_replicas: 0 - sort: - field: "@timestamp" - order: desc - composed_of: - - "event-mappings" - - "logs-elastic_agent.fleet_server@package" - - "logs-elastic_agent.fleet_server@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-elastic_agent_x_heartbeat: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.heartbeat-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - composed_of: - - "logs-elastic_agent.heartbeat@package" - - "logs-elastic_agent.heartbeat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-elastic_agent: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - composed_of: - - "event-mappings" - - "logs-elastic_agent@package" - - "logs-elastic_agent@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-elastic_agent_x_metricbeat: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.metricbeat-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - composed_of: - - "event-mappings" - - "logs-elastic_agent.metricbeat@package" - - "logs-elastic_agent.metricbeat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-elastic_agent_x_osquerybeat: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.osquerybeat-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - composed_of: - - "event-mappings" - - "logs-elastic_agent.osquerybeat@package" - - "logs-elastic_agent.osquerybeat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - so-logs-elastic_agent_x_packetbeat: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.packetbeat-*" - template: - settings: - index: - number_of_replicas: 0 - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true - composed_of: - - "logs-elastic_agent.packetbeat@package" - - "logs-elastic_agent.packetbeat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb - cold: - min_age: 30d - actions: - set_priority: - priority: 0 - delete: - min_age: 365d - actions: - delete: {} - _meta: - package: - name: elastic_agent - managed_by: security_onion - managed: true so-case: - index_sorting: False + index_sorting: false index_template: + composed_of: + - case-mappings + - case-settings index_patterns: - - so-case* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 1500 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - case-mappings - - case-settings + - so-case* priority: 500 - so-common: - warm: 7 - close: 30 - delete: 365 - index_sorting: False - index_template: - data_stream: {} - index_patterns: - - logs-*-so* template: mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - syslog-mappings - - dtc-syslog-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - - winlog-mappings - priority: 1 - so-endgame: - index_sorting: False - index_template: - index_patterns: - - endgame* - template: - mappings: dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - endgame-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - - winlog-mappings - priority: 500 - so-idh: - warm: 7 - close: 30 - delete: 365 - index_sorting: False - index_template: - index_patterns: - - so-idh-* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 - number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - container-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - common-settings - - common-dynamic-mappings - priority: 500 - so-suricata: - index_sorting: False - index_template: - data_stream: {} - index_patterns: - - logs-suricata-so* - template: - mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - date_detection: false + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string settings: index: lifecycle: - name: so-suricata-logs + name: so-case-logs mapping: total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 + limit: 1500 number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - suricata-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc policy: phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb cold: - min_age: 30d actions: set_priority: priority: 0 + min_age: 30d delete: - min_age: 365d actions: delete: {} - so-import: - index_sorting: False + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-common: + close: 30 + delete: 365 + index_sorting: false index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - cloud-mappings + - container-mappings + - data_stream-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - orchestrator-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - registry-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - syslog-mappings + - dtc-syslog-mappings + - threat-mappings + - tls-mappings + - tracing-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - vulnerability-mappings + - common-settings + - common-dynamic-mappings + - winlog-mappings data_stream: {} index_patterns: - - logs-import-so* + - logs-*-so* + priority: 1 template: mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string settings: index: + lifecycle: + name: so-common-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + warm: 7 + so-endgame: + index_sorting: false + index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - cloud-mappings + - container-mappings + - data_stream-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - endgame-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - orchestrator-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - registry-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - threat-mappings + - tls-mappings + - tracing-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - vulnerability-mappings + - common-settings + - common-dynamic-mappings + - winlog-mappings + index_patterns: + - endgame* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + lifecycle: + name: so-endgame-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-idh: + close: 30 + delete: 365 + index_sorting: false + index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - container-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - threat-mappings + - tls-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - common-settings + - common-dynamic-mappings + index_patterns: + - so-idh-* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + lifecycle: + name: so-idh-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + warm: 7 + so-import: + index_sorting: false + index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - cloud-mappings + - container-mappings + - data_stream-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - orchestrator-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - registry-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - threat-mappings + - tls-mappings + - tracing-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - vulnerability-mappings + - common-settings + - common-dynamic-mappings + - winlog-mappings + data_stream: {} + index_patterns: + - logs-import-so* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + final_pipeline: .fleet_final_pipeline-1 lifecycle: name: so-import-logs mapping: total_fields: limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 number_of_replicas: 0 - final_pipeline: ".fleet_final_pipeline-1" - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - - winlog-mappings - priority: 500 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc policy: phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb cold: - min_age: 30d actions: set_priority: priority: 0 + min_age: 30d delete: - min_age: 365d actions: delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-kratos: - warm: 7 close: 30 delete: 365 - index_sorting: False + index_sorting: false index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - container-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - threat-mappings + - tls-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - common-settings + - common-dynamic-mappings data_stream: - hidden: false allow_custom_routing: false + hidden: false index_patterns: - - logs-kratos-so* + - logs-kratos-so* + priority: 500 template: mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string settings: index: + lifecycle: + name: so-kratos-logs mapping: total_fields: limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - container-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - common-settings - - common-dynamic-mappings - priority: 500 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc policy: phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb cold: - min_age: 30d actions: set_priority: priority: 0 + min_age: 30d delete: - min_age: 365d actions: delete: {} - so-logstash: - index_sorting: False + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + warm: 7 + so-logs: + index_sorting: false index_template: + composed_of: + - so-data-streams-mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + - so-logs-mappings + - so-logs-settings + data_stream: + allow_custom_routing: false + hidden: false index_patterns: - - logs-logstash-default* + - logs-*-* + priority: 225 + template: + mappings: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + settings: + index: + lifecycle: + name: so-logs-logs + mapping: + total_fields: + limit: 5001 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-1password_x_item_usages: + index_sorting: false + index_template: + composed_of: + - logs-1password.item_usages@package + - logs-1password.item_usages@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-1password.item_usages-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-1password.item_usages-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-1password_x_signin_attempts: + index_sorting: false + index_template: + composed_of: + - logs-1password.signin_attempts@package + - logs-1password.signin_attempts@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-1password.signin_attempts-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-1password.signin_attempts-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-apache_x_access: + index_sorting: false + index_template: + composed_of: + - logs-apache.access@package + - logs-apache.access@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-apache.access-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-apache.access-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-apache_x_error: + index_sorting: false + index_template: + composed_of: + - logs-apache.error@package + - logs-apache.error@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-apache.error-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-apache.error-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-auditd_x_log: + index_sorting: false + index_template: + composed_of: + - logs-auditd.log@package + - logs-auditd.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-auditd.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-auditd.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-auth0_x_logs: + index_sorting: false + index_template: + composed_of: + - logs-auth0.logs@package + - logs-auth0.logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-auth0.logs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-auth0.logs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-aws_x_cloudtrail: + index_sorting: false + index_template: + composed_of: + - logs-aws.cloudtrail@package + - logs-aws.cloudtrail@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-aws.cloudtrail-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-aws.cloudtrail-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-aws_x_cloudwatch_logs: + index_sorting: false + index_template: + composed_of: + - logs-aws.cloudwatch_logs@package + - logs-aws.cloudwatch_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-aws.cloudwatch_logs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-aws.cloudwatch_logs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-aws_x_ec2_logs: + index_sorting: false + index_template: + composed_of: + - logs-aws.ec2_logs@package + - logs-aws.ec2_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-aws.ec2_logs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-aws.ec2_logs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-aws_x_elb_logs: + index_sorting: false + index_template: + composed_of: + - logs-aws.elb_logs@package + - logs-aws.elb_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-aws.elb_logs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-aws.elb_logs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-aws_x_firewall_logs: + index_sorting: false + index_template: + composed_of: + - logs-aws.firewall_logs@package + - logs-aws.firewall_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-aws.firewall_logs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-aws.firewall_logs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-aws_x_route53_public_logs: + index_sorting: false + index_template: + composed_of: + - logs-aws.route53_public_logs@package + - logs-aws.route53_public_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-aws.route53_public_logs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-aws.route53_public_logs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-aws_x_route53_resolver_logs: + index_sorting: false + index_template: + composed_of: + - logs-aws.route53_resolver_logs@package + - logs-aws.route53_resolver_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-aws.route53_resolver_logs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-aws.route53_resolver_logs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-aws_x_s3access: + index_sorting: false + index_template: + composed_of: + - logs-aws.s3access@package + - logs-aws.s3access@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-aws.s3access-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-aws.s3access-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-aws_x_vpcflow: + index_sorting: false + index_template: + composed_of: + - logs-aws.vpcflow@package + - logs-aws.vpcflow@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-aws.vpcflow-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-aws.vpcflow-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-aws_x_waf: + index_sorting: false + index_template: + composed_of: + - logs-aws.waf@package + - logs-aws.waf@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-aws.waf-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-aws.waf-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-azure_x_activitylogs: + index_sorting: false + index_template: + composed_of: + - logs-azure.activitylogs@package + - logs-azure.activitylogs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-azure.activitylogs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-azure.activitylogs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-azure_x_application_gateway: + index_sorting: false + index_template: + composed_of: + - logs-azure.application_gateway@package + - logs-azure.application_gateway@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-azure.application_gateway-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-azure.application_gateway-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-azure_x_auditlogs: + index_sorting: false + index_template: + composed_of: + - logs-azure.auditlogs@package + - logs-azure.auditlogs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-azure.auditlogs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-azure.auditlogs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-azure_x_eventhub: + index_sorting: false + index_template: + composed_of: + - logs-azure.eventhub@package + - logs-azure.eventhub@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-azure.eventhub-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-azure.eventhub-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-azure_x_firewall_logs: + index_sorting: false + index_template: + composed_of: + - logs-azure.firewall_logs@package + - logs-azure.firewall_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-azure.firewall_logs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-azure.firewall_logs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-azure_x_identity_protection: + index_sorting: false + index_template: + composed_of: + - logs-azure.identity_protection@package + - logs-azure.identity_protection@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-azure.identity_protection-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-azure.identity_protection-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-azure_x_platformlogs: + index_sorting: false + index_template: + composed_of: + - logs-azure.platformlogs@package + - logs-azure.platformlogs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-azure.platformlogs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-azure.platformlogs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-azure_x_provisioning: + index_sorting: false + index_template: + composed_of: + - logs-azure.provisioning@package + - logs-azure.provisioning@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-azure.provisioning-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-azure.provisioning-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-azure_x_signinlogs: + index_sorting: false + index_template: + composed_of: + - logs-azure.signinlogs@package + - logs-azure.signinlogs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-azure.signinlogs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-azure.signinlogs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-azure_x_springcloudlogs: + index_sorting: false + index_template: + composed_of: + - logs-azure.springcloudlogs@package + - logs-azure.springcloudlogs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-azure.springcloudlogs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-azure.springcloudlogs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-barracuda_x_waf: + index_sorting: false + index_template: + composed_of: + - logs-barracuda.waf@package + - logs-barracuda.waf@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-barracuda.waf-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-barracuda.waf-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-carbonblack_edr_x_log: + index_sorting: false + index_template: + composed_of: + - logs-carbonblack_edr.log@package + - logs-carbonblack_edr.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-carbonblack_edr.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-carbonblack_edr.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-cisco_asa_x_log: + index_sorting: false + index_template: + composed_of: + - logs-cisco_asa.log@package + - logs-cisco_asa.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-cisco_asa.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-cisco_asa.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-cisco_duo_x_admin: + index_sorting: false + index_template: + composed_of: + - logs-cisco_duo.admin@package + - logs-cisco_duo.admin@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-cisco_duo.admin-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-cisco_duo.admin-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-cisco_duo_x_auth: + index_sorting: false + index_template: + composed_of: + - logs-cisco_duo.auth@package + - logs-cisco_duo.auth@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-cisco_duo.auth-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-cisco_duo.auth-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-cisco_duo_x_offline_enrollment: + index_sorting: false + index_template: + composed_of: + - logs-cisco_duo.offline_enrollment@package + - logs-cisco_duo.offline_enrollment@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-cisco_duo.offline_enrollment-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-cisco_duo.offline_enrollment-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-cisco_duo_x_summary: + index_sorting: false + index_template: + composed_of: + - logs-cisco_duo.summary@package + - logs-cisco_duo.summary@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-cisco_duo.summary-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-cisco_duo.summary-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-cisco_duo_x_telephony: + index_sorting: false + index_template: + composed_of: + - logs-cisco_duo.telephony@package + - logs-cisco_duo.telephony@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-cisco_duo.telephony-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-cisco_duo.telephony-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-cisco_meraki_x_events: + index_sorting: false + index_template: + composed_of: + - logs-cisco_meraki.events@package + - logs-cisco_meraki.events@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-cisco_meraki.events-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-cisco_meraki.events-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-cisco_meraki_x_log: + index_sorting: false + index_template: + composed_of: + - logs-cisco_meraki.log@package + - logs-cisco_meraki.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-cisco_meraki.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-cisco_meraki.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-cisco_umbrella_x_log: + index_sorting: false + index_template: + composed_of: + - logs-cisco_umbrella.log@package + - logs-cisco_umbrella.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-cisco_umbrella.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-cisco_umbrella.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-cloudflare_x_audit: + index_sorting: false + index_template: + composed_of: + - logs-cloudflare.audit@package + - logs-cloudflare.audit@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-cloudflare.audit-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-cloudflare.audit-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-cloudflare_x_logpull: + index_sorting: false + index_template: + composed_of: + - logs-cloudflare.logpull@package + - logs-cloudflare.logpull@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-cloudflare.logpull-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-cloudflare.logpull-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-crowdstrike_x_falcon: + index_sorting: false + index_template: + composed_of: + - logs-crowdstrike.falcon@package + - logs-crowdstrike.falcon@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-crowdstrike.falcon-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-crowdstrike.falcon-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-crowdstrike_x_fdr: + index_sorting: false + index_template: + composed_of: + - logs-crowdstrike.fdr@package + - logs-crowdstrike.fdr@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-crowdstrike.fdr-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-crowdstrike.fdr-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-darktrace_x_ai_analyst_alert: + index_sorting: false + index_template: + composed_of: + - logs-darktrace.ai_analyst_alert@package + - logs-darktrace.ai_analyst_alert@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-darktrace.ai_analyst_alert-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-darktrace.ai_analyst_alert-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-darktrace_x_model_breach_alert: + index_sorting: false + index_template: + composed_of: + - logs-darktrace.model_breach_alert@package + - logs-darktrace.model_breach_alert@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-darktrace.model_breach_alert-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-darktrace.model_breach_alert-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-darktrace_x_system_status_alert: + index_sorting: false + index_template: + composed_of: + - logs-darktrace.system_status_alert@package + - logs-darktrace.system_status_alert@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-darktrace.system_status_alert-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-darktrace.system_status_alert-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-elastic_agent: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-elastic_agent@package + - logs-elastic_agent@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-elastic_agent-* + priority: 501 + template: + mappings: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + settings: + index: + lifecycle: + name: so-logs-elastic_agent-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-elastic_agent_x_apm_server: + index_sorting: false + index_template: + composed_of: + - logs-elastic_agent.apm_server@package + - logs-elastic_agent.apm_server@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-elastic_agent.apm_server-* + priority: 501 + template: + mappings: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + settings: + index: + lifecycle: + name: so-logs-elastic_agent.apm_server-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-elastic_agent_x_auditbeat: + index_sorting: false + index_template: + composed_of: + - logs-elastic_agent.auditbeat@package + - logs-elastic_agent.auditbeat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-elastic_agent.auditbeat-* + priority: 501 + template: + mappings: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + settings: + index: + lifecycle: + name: so-logs-elastic_agent.auditbeat-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-elastic_agent_x_cloudbeat: + index_sorting: false + index_template: + composed_of: + - logs-elastic_agent.cloudbeat@package + - logs-elastic_agent.cloudbeat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + index_patterns: + - logs-elastic_agent.cloudbeat-* + priority: 501 + template: + mappings: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + settings: + index: + lifecycle: + name: so-logs-elastic_agent.cloudbeat-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-elastic_agent_x_endpoint_security: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-elastic_agent.endpoint_security@package + - logs-elastic_agent.endpoint_security@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-elastic_agent.endpoint_security-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-elastic_agent.endpoint_security-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-elastic_agent_x_filebeat: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-elastic_agent.filebeat@package + - logs-elastic_agent.filebeat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-elastic_agent.filebeat-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-elastic_agent.filebeat-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-elastic_agent_x_fleet_server: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-elastic_agent.fleet_server@package + - logs-elastic_agent.fleet_server@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-elastic_agent.fleet_server-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-elastic_agent.fleet_server-logs + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-elastic_agent_x_heartbeat: + index_sorting: false + index_template: + composed_of: + - logs-elastic_agent.heartbeat@package + - logs-elastic_agent.heartbeat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + index_patterns: + - logs-elastic_agent.heartbeat-* + priority: 501 + template: + mappings: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + settings: + index: + lifecycle: + name: so-logs-elastic_agent.heartbeat-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-elastic_agent_x_metricbeat: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-elastic_agent.metricbeat@package + - logs-elastic_agent.metricbeat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-elastic_agent.metricbeat-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-elastic_agent.metricbeat-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-elastic_agent_x_osquerybeat: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-elastic_agent.osquerybeat@package + - logs-elastic_agent.osquerybeat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-elastic_agent.osquerybeat-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-elastic_agent.osquerybeat-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-elastic_agent_x_packetbeat: + index_sorting: false + index_template: + composed_of: + - logs-elastic_agent.packetbeat@package + - logs-elastic_agent.packetbeat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-elastic_agent.packetbeat-* + priority: 501 + template: + mappings: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + settings: + index: + lifecycle: + name: so-logs-elastic_agent.packetbeat-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-endpoint_x_alerts: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-endpoint.alerts@custom + - logs-endpoint.alerts@package + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-endpoint.alerts-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-endpoint.alerts-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-endpoint_x_events_x_api: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-endpoint.events.api@custom + - logs-endpoint.events.api@package + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-endpoint.events.api-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-endpoint.events.api-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-endpoint_x_events_x_file: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-endpoint.events.file@custom + - logs-endpoint.events.file@package + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-endpoint.events.file-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-endpoint.events.file-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-endpoint_x_events_x_library: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-endpoint.events.library@custom + - logs-endpoint.events.library@package + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-endpoint.events.library-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-endpoint.events.library-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-endpoint_x_events_x_network: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-endpoint.events.network@custom + - logs-endpoint.events.network@package + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-endpoint.events.network-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-endpoint.events.network-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-endpoint_x_events_x_process: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-endpoint.events.process@custom + - logs-endpoint.events.process@package + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-endpoint.events.process-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-endpoint.events.process-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-endpoint_x_events_x_registry: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-endpoint.events.registry@custom + - logs-endpoint.events.registry@package + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-endpoint.events.registry-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-endpoint.events.registry-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-endpoint_x_events_x_security: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-endpoint.events.security@custom + - logs-endpoint.events.security@package + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-endpoint.events.security-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-endpoint.events.security-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-f5_bigip_x_log: + index_sorting: false + index_template: + composed_of: + - logs-f5_bigip.log@package + - logs-f5_bigip.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-f5_bigip.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-f5_bigip.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-fim_x_event: + index_sorting: false + index_template: + composed_of: + - logs-fim.event@package + - logs-fim.event@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-fim.event-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-fim.event-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-fireeye_x_nx: + index_sorting: false + index_template: + composed_of: + - logs-fireeye.nx@package + - logs-fireeye.nx@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-fireeye.nx-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-fireeye.nx-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-fortinet_fortigate_x_log: + index_sorting: false + index_template: + composed_of: + - logs-fortinet_fortigate.log@package + - logs-fortinet_fortigate.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-fortinet_fortigate.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-fortinet_fortigate.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-fortinet_x_clientendpoint: + index_sorting: false + index_template: + composed_of: + - logs-fortinet.clientendpoint@package + - logs-fortinet.clientendpoint@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-fortinet.clientendpoint-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-fortinet.clientendpoint-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-fortinet_x_firewall: + index_sorting: false + index_template: + composed_of: + - logs-fortinet.firewall@package + - logs-fortinet.firewall@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-fortinet.firewall-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-fortinet.firewall-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-fortinet_x_fortimail: + index_sorting: false + index_template: + composed_of: + - logs-fortinet.fortimail@package + - logs-fortinet.fortimail@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-fortinet.fortimail-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-fortinet.fortimail-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-fortinet_x_fortimanager: + index_sorting: false + index_template: + composed_of: + - logs-fortinet.fortimanager@package + - logs-fortinet.fortimanager@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-fortinet.fortimanager-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-fortinet.fortimanager-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-gcp_x_audit: + index_sorting: false + index_template: + composed_of: + - logs-gcp.audit@package + - logs-gcp.audit@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-gcp.audit-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-gcp.audit-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-gcp_x_dns: + index_sorting: false + index_template: + composed_of: + - logs-gcp.dns@package + - logs-gcp.dns@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-gcp.dns-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-gcp.dns-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-gcp_x_firewall: + index_sorting: false + index_template: + composed_of: + - logs-gcp.firewall@package + - logs-gcp.firewall@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-gcp.firewall-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-gcp.firewall-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-gcp_x_loadbalancing_logs: + index_sorting: false + index_template: + composed_of: + - logs-gcp.loadbalancing_logs@package + - logs-gcp.loadbalancing_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-gcp.loadbalancing_logs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-gcp.loadbalancing_logs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-gcp_x_vpcflow: + index_sorting: false + index_template: + composed_of: + - logs-gcp.vpcflow@package + - logs-gcp.vpcflow@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-gcp.vpcflow-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-gcp.vpcflow-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-github_x_audit: + index_sorting: false + index_template: + composed_of: + - logs-github.audit@package + - logs-github.audit@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-github.audit-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-github.audit-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-github_x_code_scanning: + index_sorting: false + index_template: + composed_of: + - logs-github.code_scanning@package + - logs-github.code_scanning@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-github.code_scanning-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-github.code_scanning-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-github_x_dependabot: + index_sorting: false + index_template: + composed_of: + - logs-github.dependabot@package + - logs-github.dependabot@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-github.dependabot-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-github.dependabot-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-github_x_issues: + index_sorting: false + index_template: + composed_of: + - logs-github.issues@package + - logs-github.issues@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-github.issues-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-github.issues-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-github_x_secret_scanning: + index_sorting: false + index_template: + composed_of: + - logs-github.secret_scanning@package + - logs-github.secret_scanning@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-github.secret_scanning-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-github.secret_scanning-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-google_workspace_x_access_transparency: + index_sorting: false + index_template: + composed_of: + - logs-google_workspace.access_transparency@package + - logs-google_workspace.access_transparency@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-google_workspace.access_transparency-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-google_workspace.access_transparency-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-google_workspace_x_admin: + index_sorting: false + index_template: + composed_of: + - logs-google_workspace.admin@package + - logs-google_workspace.admin@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-google_workspace.admin-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-google_workspace.admin-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-google_workspace_x_alert: + index_sorting: false + index_template: + composed_of: + - logs-google_workspace.alert@package + - logs-google_workspace.alert@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-google_workspace.alert-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-google_workspace.alert-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-google_workspace_x_context_aware_access: + index_sorting: false + index_template: + composed_of: + - logs-google_workspace.context_aware_access@package + - logs-google_workspace.context_aware_access@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-google_workspace.context_aware_access-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-google_workspace.context_aware_access-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-google_workspace_x_device: + index_sorting: false + index_template: + composed_of: + - logs-google_workspace.device@package + - logs-google_workspace.device@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-google_workspace.device-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-google_workspace.device-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-google_workspace_x_drive: + index_sorting: false + index_template: + composed_of: + - logs-google_workspace.drive@package + - logs-google_workspace.drive@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-google_workspace.drive-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-google_workspace.drive-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-google_workspace_x_gcp: + index_sorting: false + index_template: + composed_of: + - logs-google_workspace.gcp@package + - logs-google_workspace.gcp@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-google_workspace.gcp-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-google_workspace.gcp-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-google_workspace_x_group_enterprise: + index_sorting: false + index_template: + composed_of: + - logs-google_workspace.group_enterprise@package + - logs-google_workspace.group_enterprise@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-google_workspace.group_enterprise-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-google_workspace.group_enterprise-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-google_workspace_x_groups: + index_sorting: false + index_template: + composed_of: + - logs-google_workspace.groups@package + - logs-google_workspace.groups@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-google_workspace.groups-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-google_workspace.groups-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-google_workspace_x_login: + index_sorting: false + index_template: + composed_of: + - logs-google_workspace.login@package + - logs-google_workspace.login@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-google_workspace.login-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-google_workspace.login-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-google_workspace_x_rules: + index_sorting: false + index_template: + composed_of: + - logs-google_workspace.rules@package + - logs-google_workspace.rules@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-google_workspace.rules-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-google_workspace.rules-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-google_workspace_x_saml: + index_sorting: false + index_template: + composed_of: + - logs-google_workspace.saml@package + - logs-google_workspace.saml@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-google_workspace.saml-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-google_workspace.saml-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-google_workspace_x_token: + index_sorting: false + index_template: + composed_of: + - logs-google_workspace.token@package + - logs-google_workspace.token@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-google_workspace.token-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-google_workspace.token-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-google_workspace_x_user_accounts: + index_sorting: false + index_template: + composed_of: + - logs-google_workspace.user_accounts@package + - logs-google_workspace.user_accounts@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-google_workspace.user_accounts-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-google_workspace.user_accounts-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-http_endpoint_x_generic: + index_sorting: false + index_template: + composed_of: + - logs-http_endpoint.generic@package + - logs-http_endpoint.generic@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-http_endpoint.generic-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-http_endpoint.generic-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-httpjson_x_generic: + index_sorting: false + index_template: + composed_of: + - logs-httpjson.generic@package + - logs-httpjson.generic@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-httpjson.generic-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-httpjson.generic-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-juniper_srx_x_log: + index_sorting: false + index_template: + composed_of: + - logs-juniper_srx.log@package + - logs-juniper_srx.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-juniper_srx.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-juniper_srx.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-juniper_x_junos: + index_sorting: false + index_template: + composed_of: + - logs-juniper.junos@package + - logs-juniper.junos@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-juniper.junos-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-juniper.junos-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-juniper_x_netscreen: + index_sorting: false + index_template: + composed_of: + - logs-juniper.netscreen@package + - logs-juniper.netscreen@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-juniper.netscreen-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-juniper.netscreen-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-juniper_x_srx: + index_sorting: false + index_template: + composed_of: + - logs-juniper.srx@package + - logs-juniper.srx@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-juniper.srx-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-juniper.srx-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-kafka_log_x_generic: + index_sorting: false + index_template: + composed_of: + - logs-kafka_log.generic@package + - logs-kafka_log.generic@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-kafka_log.generic-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-kafka_log.generic-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-lastpass_x_detailed_shared_folder: + index_sorting: false + index_template: + composed_of: + - logs-lastpass.detailed_shared_folder@package + - logs-lastpass.detailed_shared_folder@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-lastpass.detailed_shared_folder-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-lastpass.detailed_shared_folder-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-lastpass_x_event_report: + index_sorting: false + index_template: + composed_of: + - logs-lastpass.event_report@package + - logs-lastpass.event_report@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-lastpass.event_report-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-lastpass.event_report-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-lastpass_x_user: + index_sorting: false + index_template: + composed_of: + - logs-lastpass.user@package + - logs-lastpass.user@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-lastpass.user-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-lastpass.user-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-m365_defender_x_event: + index_sorting: false + index_template: + composed_of: + - logs-m365_defender.event@package + - logs-m365_defender.event@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-m365_defender.event-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-m365_defender.event-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-m365_defender_x_incident: + index_sorting: false + index_template: + composed_of: + - logs-m365_defender.incident@package + - logs-m365_defender.incident@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-m365_defender.incident-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-m365_defender.incident-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-m365_defender_x_log: + index_sorting: false + index_template: + composed_of: + - logs-m365_defender.log@package + - logs-m365_defender.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-m365_defender.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-m365_defender.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-microsoft_defender_endpoint_x_log: + index_sorting: false + index_template: + composed_of: + - logs-microsoft_defender_endpoint.log@package + - logs-microsoft_defender_endpoint.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-microsoft_defender_endpoint.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-microsoft_defender_endpoint.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-microsoft_dhcp_x_log: + index_sorting: false + index_template: + composed_of: + - logs-microsoft_dhcp.log@package + - logs-microsoft_dhcp.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-microsoft_dhcp.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-microsoft_dhcp.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-mimecast_x_audit_events: + index_sorting: false + index_template: + composed_of: + - logs-mimecast.audit_events@package + - logs-mimecast.audit_events@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-mimecast.audit_events-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-mimecast.audit_events-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-mimecast_x_dlp_logs: + index_sorting: false + index_template: + composed_of: + - logs-mimecast.dlp_logs@package + - logs-mimecast.dlp_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-mimecast.dlp_logs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-mimecast.dlp_logs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-mimecast_x_siem_logs: + index_sorting: false + index_template: + composed_of: + - logs-mimecast.siem_logs@package + - logs-mimecast.siem_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-mimecast.siem_logs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-mimecast.siem_logs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-mimecast_x_threat_intel_malware_customer: + index_sorting: false + index_template: + composed_of: + - logs-mimecast.threat_intel_malware_customer@package + - logs-mimecast.threat_intel_malware_customer@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-mimecast.threat_intel_malware_customer-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-mimecast.threat_intel_malware_customer-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-mimecast_x_threat_intel_malware_grid: + index_sorting: false + index_template: + composed_of: + - logs-mimecast.threat_intel_malware_grid@package + - logs-mimecast.threat_intel_malware_grid@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-mimecast.threat_intel_malware_grid-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-mimecast.threat_intel_malware_grid-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-mimecast_x_ttp_ap_logs: + index_sorting: false + index_template: + composed_of: + - logs-mimecast.ttp_ap_logs@package + - logs-mimecast.ttp_ap_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-mimecast.ttp_ap_logs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-mimecast.ttp_ap_logs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-mimecast_x_ttp_ip_logs: + index_sorting: false + index_template: + composed_of: + - logs-mimecast.ttp_ip_logs@package + - logs-mimecast.ttp_ip_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-mimecast.ttp_ip_logs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-mimecast.ttp_ip_logs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-mimecast_x_ttp_url_logs: + index_sorting: false + index_template: + composed_of: + - logs-mimecast.ttp_url_logs@package + - logs-mimecast.ttp_url_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-mimecast.ttp_url_logs-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-mimecast.ttp_url_logs-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-netflow_x_log: + index_sorting: false + index_template: + composed_of: + - logs-netflow.log@package + - logs-netflow.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-netflow.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-netflow.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-o365_x_audit: + index_sorting: false + index_template: + composed_of: + - logs-o365.audit@package + - logs-o365.audit@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-o365.audit-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-o365.audit-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-okta_x_system: + index_sorting: false + index_template: + composed_of: + - logs-okta.system@package + - logs-okta.system@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-okta.system-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-okta.system-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-osquery-manager-action_x_responses: + index_sorting: false + index_template: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + composed_of: + - logs-osquery_manager.action.responses + index_patterns: + - .logs-osquery_manager.action.responses* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-osquery-manager-action.responses-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-osquery-manager-actions: + index_sorting: false + index_template: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + composed_of: + - logs-osquery_manager.actions + index_patterns: + - .logs-osquery_manager.actions* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-osquery-manager-actions-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-panw_x_panos: + index_sorting: false + index_template: + composed_of: + - logs-panw.panos@package + - logs-panw.panos@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-panw.panos-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-panw.panos-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-pfsense_x_log: + index_sorting: false + index_template: + composed_of: + - logs-pfsense.log@package + - logs-pfsense.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-pfsense.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-pfsense.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-pulse_connect_secure_x_log: + index_sorting: false + index_template: + composed_of: + - logs-pulse_connect_secure.log@package + - logs-pulse_connect_secure.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-pulse_connect_secure.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-pulse_connect_secure.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-sentinel_one_x_activity: + index_sorting: false + index_template: + composed_of: + - logs-sentinel_one.activity@package + - logs-sentinel_one.activity@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-sentinel_one.activity-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-sentinel_one.activity-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-sentinel_one_x_agent: + index_sorting: false + index_template: + composed_of: + - logs-sentinel_one.agent@package + - logs-sentinel_one.agent@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-sentinel_one.agent-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-sentinel_one.agent-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-sentinel_one_x_alert: + index_sorting: false + index_template: + composed_of: + - logs-sentinel_one.alert@package + - logs-sentinel_one.alert@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-sentinel_one.alert-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-sentinel_one.alert-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-sentinel_one_x_group: + index_sorting: false + index_template: + composed_of: + - logs-sentinel_one.group@package + - logs-sentinel_one.group@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-sentinel_one.group-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-sentinel_one.group-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-sentinel_one_x_threat: + index_sorting: false + index_template: + composed_of: + - logs-sentinel_one.threat@package + - logs-sentinel_one.threat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-sentinel_one.threat-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-sentinel_one.threat-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-snyk_x_audit: + index_sorting: false + index_template: + composed_of: + - logs-snyk.audit@package + - logs-snyk.audit@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-snyk.audit-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-snyk.audit-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-snyk_x_vulnerabilities: + index_sorting: false + index_template: + composed_of: + - logs-snyk.vulnerabilities@package + - logs-snyk.vulnerabilities@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-snyk.vulnerabilities-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-snyk.vulnerabilities-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-sonicwall_firewall_x_log: + index_sorting: false + index_template: + composed_of: + - logs-sonicwall_firewall.log@package + - logs-sonicwall_firewall.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-sonicwall_firewall.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-sonicwall_firewall.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-sophos_central_x_alert: + index_sorting: false + index_template: + composed_of: + - logs-sophos_central.alert@package + - logs-sophos_central.alert@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-sophos_central.alert-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-sophos_central.alert-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-sophos_central_x_event: + index_sorting: false + index_template: + composed_of: + - logs-sophos_central.event@package + - logs-sophos_central.event@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-sophos_central.event-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-sophos_central.event-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-sophos_x_utm: + index_sorting: false + index_template: + composed_of: + - logs-sophos.utm@package + - logs-sophos.utm@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-sophos.utm-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-sophos.utm-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-sophos_x_xg: + index_sorting: false + index_template: + composed_of: + - logs-sophos.xg@package + - logs-sophos.xg@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-sophos.xg-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-sophos.xg-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-symantec_endpoint_x_log: + index_sorting: false + index_template: + composed_of: + - logs-symantec_endpoint.log@package + - logs-symantec_endpoint.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-symantec_endpoint.log-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-symantec_endpoint.log-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-system_x_application: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-system.application@package + - logs-system.application@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-system.application* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-system.application-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-system_x_auth: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-system.auth@package + - logs-system.auth@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-system.auth* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-system.auth-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-system_x_security: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-system.security@package + - logs-system.security@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-system.security* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-system.security-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-system_x_syslog: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-system.syslog@package + - logs-system.syslog@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-system.syslog* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-system.syslog-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-system_x_system: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-system.system@package + - logs-system.system@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-system.system* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-system.system-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-tenable_sc_x_asset: + index_sorting: false + index_template: + composed_of: + - logs-tenable_sc.asset@package + - logs-tenable_sc.asset@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-tenable_sc.asset-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-tenable_sc.asset-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-tenable_sc_x_plugin: + index_sorting: false + index_template: + composed_of: + - logs-tenable_sc.plugin@package + - logs-tenable_sc.plugin@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-tenable_sc.plugin-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-tenable_sc.plugin-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-tenable_sc_x_vulnerability: + index_sorting: false + index_template: + composed_of: + - logs-tenable_sc.vulnerability@package + - logs-tenable_sc.vulnerability@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-tenable_sc.vulnerability-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-tenable_sc.vulnerability-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-ti_abusech_x_malware: + index_sorting: false + index_template: + composed_of: + - logs-ti_abusech.malware@package + - logs-ti_abusech.malware@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-ti_abusech.malware-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-ti_abusech.malware-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-ti_abusech_x_malwarebazaar: + index_sorting: false + index_template: + composed_of: + - logs-ti_abusech.malwarebazaar@package + - logs-ti_abusech.malwarebazaar@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-ti_abusech.malwarebazaar-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-ti_abusech.malwarebazaar-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-ti_abusech_x_threatfox: + index_sorting: false + index_template: + composed_of: + - logs-ti_abusech.threatfox@package + - logs-ti_abusech.threatfox@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-ti_abusech.threatfox-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-ti_abusech.threatfox-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-ti_abusech_x_url: + index_sorting: false + index_template: + composed_of: + - logs-ti_abusech.url@package + - logs-ti_abusech.url@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-ti_abusech.url-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-ti_abusech.url-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-ti_misp_x_threat: + index_sorting: false + index_template: + composed_of: + - logs-ti_misp.threat@package + - logs-ti_misp.threat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-ti_misp.threat-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-ti_misp.threat-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-ti_misp_x_threat_attributes: + index_sorting: false + index_template: + composed_of: + - logs-ti_misp.threat_attributes@package + - logs-ti_misp.threat_attributes@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-ti_misp.threat_attributes-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-ti_misp.threat_attributes-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-ti_otx_x_threat: + index_sorting: false + index_template: + composed_of: + - logs-ti_otx.threat@package + - logs-ti_otx.threat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-ti_otx.threat-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-ti_otx.threat-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-ti_recordedfuture_x_latest_ioc-template: + index_sorting: false + index_template: + composed_of: + - logs-ti_recordedfuture.latest_ioc-template@package + - logs-ti_recordedfuture.latest_ioc-template@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-ti_recordedfuture.latest_ioc-template-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-ti_recordedfuture.latest_ioc-template-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-ti_recordedfuture_x_threat: + index_sorting: false + index_template: + composed_of: + - logs-ti_recordedfuture.threat@package + - logs-ti_recordedfuture.threat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-ti_recordedfuture.threat-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-ti_recordedfuture.threat-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-windows_x_forwarded: + index_sorting: false + index_template: + composed_of: + - logs-windows.forwarded@package + - logs-windows.forwarded@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-windows.forwarded* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-windows.forwarded-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-windows_x_powershell: + index_sorting: false + index_template: + composed_of: + - logs-windows.powershell@package + - logs-windows.powershell@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-windows.powershell-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-windows.powershell-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-windows_x_powershell_operational: + index_sorting: false + index_template: + composed_of: + - logs-windows.powershell_operational@package + - logs-windows.powershell_operational@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-windows.powershell_operational-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-windows.powershell_operational-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-windows_x_sysmon_operational: + index_sorting: false + index_template: + composed_of: + - logs-windows.sysmon_operational@package + - logs-windows.sysmon_operational@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-windows.sysmon_operational-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-windows.sysmon_operational-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-zscaler_zia_x_alerts: + index_sorting: false + index_template: + composed_of: + - logs-zscaler_zia.alerts@package + - logs-zscaler_zia.alerts@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-zscaler_zia.alerts-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-zscaler_zia.alerts-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-zscaler_zia_x_dns: + index_sorting: false + index_template: + composed_of: + - logs-zscaler_zia.dns@package + - logs-zscaler_zia.dns@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-zscaler_zia.dns-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-zscaler_zia.dns-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-zscaler_zia_x_firewall: + index_sorting: false + index_template: + composed_of: + - logs-zscaler_zia.firewall@package + - logs-zscaler_zia.firewall@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-zscaler_zia.firewall-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-zscaler_zia.firewall-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-zscaler_zia_x_tunnel: + index_sorting: false + index_template: + composed_of: + - logs-zscaler_zia.tunnel@package + - logs-zscaler_zia.tunnel@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-zscaler_zia.tunnel-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-zscaler_zia.tunnel-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-zscaler_zia_x_web: + index_sorting: false + index_template: + composed_of: + - logs-zscaler_zia.web@package + - logs-zscaler_zia.web@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-zscaler_zia.web-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-zscaler_zia.web-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-zscaler_zpa_x_app_connector_status: + index_sorting: false + index_template: + composed_of: + - logs-zscaler_zpa.app_connector_status@package + - logs-zscaler_zpa.app_connector_status@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-zscaler_zpa.app_connector_status-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-zscaler_zpa.app_connector_status-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-zscaler_zpa_x_audit: + index_sorting: false + index_template: + composed_of: + - logs-zscaler_zpa.audit@package + - logs-zscaler_zpa.audit@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-zscaler_zpa.audit-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-zscaler_zpa.audit-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-zscaler_zpa_x_browser_access: + index_sorting: false + index_template: + composed_of: + - logs-zscaler_zpa.browser_access@package + - logs-zscaler_zpa.browser_access@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-zscaler_zpa.browser_access-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-zscaler_zpa.browser_access-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-zscaler_zpa_x_user_activity: + index_sorting: false + index_template: + composed_of: + - logs-zscaler_zpa.user_activity@package + - logs-zscaler_zpa.user_activity@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-zscaler_zpa.user_activity-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-zscaler_zpa.user_activity-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-zscaler_zpa_x_user_status: + index_sorting: false + index_template: + composed_of: + - logs-zscaler_zpa.user_status@package + - logs-zscaler_zpa.user_status@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-zscaler_zpa.user_status-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-zscaler_zpa.user_status-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logstash: + index_sorting: false + index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - cloud-mappings + - container-mappings + - data_stream-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - logstash-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - orchestrator-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - registry-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - threat-mappings + - tls-mappings + - tracing-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - vulnerability-mappings + - common-settings + - common-dynamic-mappings + index_patterns: + - logs-logstash-default* + priority: 500 template: mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string settings: index: lifecycle: @@ -3891,104 +8572,109 @@ elasticsearch: mapping: total_fields: limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - logstash-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc policy: phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb cold: - min_age: 30d actions: set_priority: priority: 0 + min_age: 30d delete: - min_age: 365d actions: delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-redis: - index_sorting: False + index_sorting: false index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - cloud-mappings + - container-mappings + - data_stream-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - orchestrator-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - redis-mappings + - registry-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - threat-mappings + - tls-mappings + - tracing-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - vulnerability-mappings + - common-settings + - common-dynamic-mappings index_patterns: - - logs-redis-default* + - logs-redis-default* + priority: 500 template: mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string settings: index: lifecycle: @@ -3996,315 +8682,447 @@ elasticsearch: mapping: total_fields: limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - redis-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc policy: phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb cold: - min_age: 30d actions: set_priority: priority: 0 + min_age: 30d delete: - min_age: 365d actions: delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-strelka: - index_sorting: False + index_sorting: false index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - cloud-mappings + - container-mappings + - data_stream-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - so-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - orchestrator-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - registry-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - so-scan-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - threat-mappings + - tls-mappings + - tracing-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - vulnerability-mappings + - common-settings + - common-dynamic-mappings data_stream: {} index_patterns: - - logs-strelka-so* + - logs-strelka-so* + priority: 500 template: mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string settings: index: + lifecycle: + name: so-strelka-logs mapping: total_fields: limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - so-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - so-scan-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc policy: phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb cold: - min_age: 30d actions: set_priority: priority: 0 + min_age: 30d delete: - min_age: 365d actions: delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-suricata: + index_sorting: false + index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - cloud-mappings + - container-mappings + - data_stream-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - orchestrator-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - registry-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - suricata-mappings + - threat-mappings + - tls-mappings + - tracing-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - vulnerability-mappings + - common-settings + - common-dynamic-mappings + data_stream: {} + index_patterns: + - logs-suricata-so* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + lifecycle: + name: so-suricata-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-syslog: - index_sorting: False + index_sorting: false index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - cloud-mappings + - container-mappings + - data_stream-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - orchestrator-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - registry-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - syslog-mappings + - dtc-syslog-mappings + - threat-mappings + - tls-mappings + - tracing-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - vulnerability-mappings + - common-settings + - common-dynamic-mappings data_stream: {} index_patterns: - - logs-syslog-so* + - logs-syslog-so* + priority: 500 template: mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string settings: index: + lifecycle: + name: so-syslog-logs mapping: total_fields: limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 1 number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - syslog-mappings - - dtc-syslog-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - priority: 500 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc policy: phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb cold: - min_age: 30d actions: set_priority: priority: 0 + min_age: 30d delete: - min_age: 365d actions: delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-zeek: - index_sorting: False + index_sorting: false index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - cloud-mappings + - container-mappings + - data_stream-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - orchestrator-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - registry-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - syslog-mappings + - dtc-syslog-mappings + - threat-mappings + - tls-mappings + - tracing-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - vulnerability-mappings + - zeek-mappings + - common-settings + - common-dynamic-mappings data_stream: {} index_patterns: - - logs-zeek-so* + - logs-zeek-so* + priority: 500 template: mappings: - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string settings: index: lifecycle: @@ -4312,656 +9130,80 @@ elasticsearch: mapping: total_fields: limit: 5000 - sort: - field: "@timestamp" - order: desc - refresh_interval: 30s - number_of_shards: 2 number_of_replicas: 0 - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - syslog-mappings - - dtc-syslog-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - zeek-mappings - - common-settings - - common-dynamic-mappings - priority: 500 + number_of_shards: 2 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc policy: phases: - hot: - min_age: 0ms - actions: - set_priority: - priority: 100 - rollover: - max_age: 30d - max_primary_shard_size: 50gb cold: - min_age: 30d actions: set_priority: priority: 0 + min_age: 30d delete: - min_age: 365d actions: delete: {} - so-logs-auth0_x_logs: - index_sorting: False - index_template: - index_patterns: - - "logs-auth0.logs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-auth0.logs@package" - - "logs-auth0.logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-carbonblack_edr_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-carbonblack_edr.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-carbonblack_edr.log@package" - - "logs-carbonblack_edr.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-cisco_duo_x_admin: - index_sorting: False - index_template: - index_patterns: - - "logs-cisco_duo.admin-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-cisco_duo.admin@package" - - "logs-cisco_duo.admin@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-cisco_duo_x_auth: - index_sorting: False - index_template: - index_patterns: - - "logs-cisco_duo.auth-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-cisco_duo.auth@package" - - "logs-cisco_duo.auth@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-cisco_duo_x_offline_enrollment: - index_sorting: False - index_template: - index_patterns: - - "logs-cisco_duo.offline_enrollment-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-cisco_duo.offline_enrollment@package" - - "logs-cisco_duo.offline_enrollment@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-cisco_duo_x_summary: - index_sorting: False - index_template: - index_patterns: - - "logs-cisco_duo.summary-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-cisco_duo.summary@package" - - "logs-cisco_duo.summary@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-cisco_duo_x_telephony: - index_sorting: False - index_template: - index_patterns: - - "logs-cisco_duo.telephony-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-cisco_duo.telephony@package" - - "logs-cisco_duo.telephony@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-cisco_meraki_x_events: - index_sorting: False - index_template: - index_patterns: - - "logs-cisco_meraki.events-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-cisco_meraki.events@package" - - "logs-cisco_meraki.events@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-cisco_meraki_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-cisco_meraki.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-cisco_meraki.log@package" - - "logs-cisco_meraki.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-cisco_umbrella_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-cisco_umbrella.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-cisco_umbrella.log@package" - - "logs-cisco_umbrella.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-fireeye_x_nx: - index_sorting: False - index_template: - index_patterns: - - "logs-fireeye.nx-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-fireeye.nx@package" - - "logs-fireeye.nx@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-mimecast_x_audit_events: - index_sorting: False - index_template: - index_patterns: - - "logs-mimecast.audit_events-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-mimecast.audit_events@package" - - "logs-mimecast.audit_events@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-mimecast_x_dlp_logs: - index_sorting: False - index_template: - index_patterns: - - "logs-mimecast.dlp_logs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-mimecast.dlp_logs@package" - - "logs-mimecast.dlp_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-mimecast_x_siem_logs: - index_sorting: False - index_template: - index_patterns: - - "logs-mimecast.siem_logs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-mimecast.siem_logs@package" - - "logs-mimecast.siem_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-mimecast_x_threat_intel_malware_customer: - index_sorting: False - index_template: - index_patterns: - - "logs-mimecast.threat_intel_malware_customer-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-mimecast.threat_intel_malware_customer@package" - - "logs-mimecast.threat_intel_malware_customer@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-mimecast_x_threat_intel_malware_grid: - index_sorting: False - index_template: - index_patterns: - - "logs-mimecast.threat_intel_malware_grid-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-mimecast.threat_intel_malware_grid@package" - - "logs-mimecast.threat_intel_malware_grid@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-mimecast_x_ttp_ap_logs: - index_sorting: False - index_template: - index_patterns: - - "logs-mimecast.ttp_ap_logs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-mimecast.ttp_ap_logs@package" - - "logs-mimecast.ttp_ap_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-mimecast_x_ttp_ip_logs: - index_sorting: False - index_template: - index_patterns: - - "logs-mimecast.ttp_ip_logs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-mimecast.ttp_ip_logs@package" - - "logs-mimecast.ttp_ip_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-mimecast_x_ttp_url_logs: - index_sorting: False - index_template: - index_patterns: - - "logs-mimecast.ttp_url_logs-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-mimecast.ttp_url_logs@package" - - "logs-mimecast.ttp_url_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-pulse_connect_secure_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-pulse_connect_secure.log-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-pulse_connect_secure.log@package" - - "logs-pulse_connect_secure.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-snyk_x_audit: - index_sorting: False - index_template: - index_patterns: - - "logs-snyk.audit-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-snyk.audit@package" - - "logs-snyk.audit@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-snyk_x_vulnerabilities: - index_sorting: False - index_template: - index_patterns: - - "logs-snyk.vulnerabilities-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-snyk.vulnerabilities@package" - - "logs-snyk.vulnerabilities@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-sophos_x_utm: - index_sorting: False - index_template: - index_patterns: - - "logs-sophos.utm-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-sophos.utm@package" - - "logs-sophos.utm@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-sophos_x_xg: - index_sorting: False - index_template: - index_patterns: - - "logs-sophos.xg-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-sophos.xg@package" - - "logs-sophos.xg@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-sophos_central_x_alert: - index_sorting: False - index_template: - index_patterns: - - "logs-sophos_central.alert-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-sophos_central.alert@package" - - "logs-sophos_central.alert@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-sophos_central_x_event: - index_sorting: False - index_template: - index_patterns: - - "logs-sophos_central.event-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-sophos_central.event@package" - - "logs-sophos_central.event@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-tenable_sc_x_asset: - index_sorting: False - index_template: - index_patterns: - - "logs-tenable_sc.asset-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-tenable_sc.asset@package" - - "logs-tenable_sc.asset@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-tenable_sc_x_plugin: - index_sorting: False - index_template: - index_patterns: - - "logs-tenable_sc.plugin-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-tenable_sc.plugin@package" - - "logs-tenable_sc.plugin@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - so-logs-tenable_sc_x_vulnerability: - index_sorting: False - index_template: - index_patterns: - - "logs-tenable_sc.vulnerability-*" - template: - settings: - index: - number_of_replicas: 0 - composed_of: - - "logs-tenable_sc.vulnerability@package" - - "logs-tenable_sc.vulnerability@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + retention: + retention_pct: 50 so_roles: - so-manager: - config: - node: - roles: - - master - - data - - remote_cluster_client - - transform - so-managersearch: - config: - node: - roles: - - master - - data_hot - - remote_cluster_client - so-standalone: - config: - node: - roles: - - master - - data_hot - - remote_cluster_client - so-searchnode: - config: - node: - roles: - - data_hot - - ingest - so-heavynode: - config: - node: - roles: - - master - - data - - remote_cluster_client - - ingest so-eval: config: node: roles: [] + so-heavynode: + config: + node: + roles: + - master + - data + - remote_cluster_client + - ingest so-import: config: node: roles: [] + so-manager: + config: + node: + roles: + - master + - data + - remote_cluster_client + - transform + so-managersearch: + config: + node: + roles: + - master + - data_hot + - remote_cluster_client + so-searchnode: + config: + node: + roles: + - data_hot + - ingest + so-standalone: + config: + node: + roles: + - master + - data_hot + - remote_cluster_client From 2e0100fd35da3c97ad3dfd292d224cd8d0814d22 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 26 Oct 2023 12:37:55 -0400 Subject: [PATCH 18/78] Update defaults.yaml --- salt/elasticsearch/defaults.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 9aef09876..721db8d99 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -9193,6 +9193,8 @@ elasticsearch: roles: - master - data_hot + - ingest + - transform - remote_cluster_client so-searchnode: config: @@ -9200,10 +9202,13 @@ elasticsearch: roles: - data_hot - ingest + - transform so-standalone: config: node: roles: - master - data_hot + - ingest + - transform - remote_cluster_client From 6891a95254b62c1a5093e6b0d465d91975d6bba0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 26 Oct 2023 13:02:39 -0400 Subject: [PATCH 19/78] remove wait_for_salt_minion from so-functions --- setup/so-functions | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index fd5bc790f..42402ad86 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2494,20 +2494,6 @@ wait_for_file() { return 1 } -wait_for_salt_minion() { - retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$setup_log" 2>&1 || fail_setup - local attempt=0 - # each attempts would take about 15 seconds - local maxAttempts=20 - until check_salt_minion_status; do - attempt=$((attempt+1)) - if [[ $attempt -eq $maxAttempts ]]; then - fail_setup - fi - sleep 10 - done -} - verify_setup() { info "Verifying setup" set -o pipefail From 47373adad253e6960b16bc1537ed22b028c259c3 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 26 Oct 2023 13:15:40 -0400 Subject: [PATCH 20/78] Specify config.yaml in config_path. Otherwise when no influxd.bolt exists influxdb will fail to read the config file and won't create a new db. Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/influxdb/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/enabled.sls b/salt/influxdb/enabled.sls index c0733c12c..293a917cb 100644 --- a/salt/influxdb/enabled.sls +++ b/salt/influxdb/enabled.sls @@ -22,7 +22,7 @@ so-influxdb: - sobridge: - ipv4_address: {{ DOCKER.containers['so-influxdb'].ip }} - environment: - - INFLUXD_CONFIG_PATH=/conf + - INFLUXD_CONFIG_PATH=/conf/config.yaml - INFLUXDB_HTTP_LOG_ENABLED=false - DOCKER_INFLUXDB_INIT_MODE=setup - DOCKER_INFLUXDB_INIT_USERNAME=so From 7e8f3b753f27df1b0bad2925fe7509d9aeb09ee0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 26 Oct 2023 13:19:04 -0400 Subject: [PATCH 21/78] add minion name to log, update comment --- salt/common/tools/sbin/so-common | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 87f40c9d4..bfa61f1b7 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -152,12 +152,12 @@ check_salt_master_status() { return 0 } -# this is only intended to be used to check the status of the minion +# this is only intended to be used to check the status of the minion from a salt master check_salt_minion_status() { local minion="$1" local timeout="${2:-5}" local logfile="${3:-'/dev/stdout'}" - echo "Checking if the salt minion will respond to jobs" >> "$logfile" 2>&1 + echo "Checking if the salt minion: $minion will respond to jobs" >> "$logfile" 2>&1 salt "$minion" test.ping -t $timeout > /dev/null 2>&1 local status=$? if [ $status -gt 0 ]; then From b37e38e3c3f842de8345858948e2c5d6cc7cd2b2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 26 Oct 2023 16:03:58 -0400 Subject: [PATCH 22/78] Update defaults.yaml --- salt/elasticsearch/defaults.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 721db8d99..44cb0ea7d 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -9173,6 +9173,7 @@ elasticsearch: roles: - master - data + - data_hot - remote_cluster_client - ingest so-import: @@ -9192,6 +9193,7 @@ elasticsearch: node: roles: - master + - data - data_hot - ingest - transform @@ -9200,6 +9202,7 @@ elasticsearch: config: node: roles: + - data - data_hot - ingest - transform @@ -9208,6 +9211,7 @@ elasticsearch: node: roles: - master + - data - data_hot - ingest - transform From cc3ee431923c89cd41f31a46318a2ff8527ca7a8 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Fri, 27 Oct 2023 07:49:34 -0400 Subject: [PATCH 23/78] Make dirs as needed --- salt/manager/kibana.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/kibana.sls b/salt/manager/kibana.sls index f9aad3f05..17ac826c2 100644 --- a/salt/manager/kibana.sls +++ b/salt/manager/kibana.sls @@ -5,3 +5,4 @@ kibana_curl_config_distributed: - template: jinja - mode: 600 - show_changes: False + - makedirs: True \ No newline at end of file From 9fc3a730356b333bce63b33237ceb4fdf09c0256 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 27 Oct 2023 08:58:08 -0400 Subject: [PATCH 24/78] Annotation changes for warm node --- salt/elasticsearch/soc_elasticsearch.yaml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index e3d257f11..189471226 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -75,8 +75,8 @@ elasticsearch: helpLink: elasticsearch.html phases: hot: - min_age: - description: Minimum age of index. This determines when the index should be moved to the hot tier. + max_age: + description: Maximum age of index. ex. 7d - This determines when the index should be moved out of the hot tier. global: True helpLink: elasticsearch.html actions: @@ -97,19 +97,29 @@ elasticsearch: helpLink: elasticsearch.html cold: min_age: - description: Minimum age of index. This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. global: True helpLink: elasticsearch.html actions: set_priority: priority: description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + global: True + helpLink: elasticsearch.html + warm: + min_age: + description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + regex: ^[0-9]d$ + actions: + set_priority: + priority: + description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. forcedType: int global: True helpLink: elasticsearch.html delete: min_age: - description: Minimum age of index. This determines when the index should be deleted. + description: Minimum age of index. ex. 90d - This determines when the index should be deleted. global: True helpLink: elasticsearch.html so-logs: &indexSettings From ce1858fe05eedb0ab614e82809cf2e0eccf6532b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 27 Oct 2023 09:02:39 -0400 Subject: [PATCH 25/78] Annotation changes for warm node --- salt/elasticsearch/soc_elasticsearch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 189471226..8bee839c1 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -109,7 +109,7 @@ elasticsearch: warm: min_age: description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. - regex: ^[0-9]d$ + regex: ^\[0-9\]{1-5}d$ actions: set_priority: priority: From 87494f64c78cec81d5633f30b59421f511554100 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 27 Oct 2023 09:06:12 -0400 Subject: [PATCH 26/78] Annotation changes for warm node --- salt/elasticsearch/soc_elasticsearch.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 8bee839c1..5b4d63f40 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -109,7 +109,8 @@ elasticsearch: warm: min_age: description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. - regex: ^\[0-9\]{1-5}d$ + regex: ^\[0-9\]{1,5}d$ + global: True actions: set_priority: priority: From 25f1a0251f423b801fe084d83f8085c7fe787b12 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 27 Oct 2023 09:08:07 -0400 Subject: [PATCH 27/78] Annotation changes for warm node --- salt/elasticsearch/soc_elasticsearch.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 5b4d63f40..a5170b776 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -110,6 +110,7 @@ elasticsearch: min_age: description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. regex: ^\[0-9\]{1,5}d$ + forcedType: string global: True actions: set_priority: From 4bbcc5002ad46d9cfe63071f9f5fcec756e4e9e3 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 25 Oct 2023 11:01:13 -0400 Subject: [PATCH 28/78] Revert "Revert "Upgrade/salt3006.3"" This reverts commit c41e19ad0b301f1deae35365e62544e987045fdd. --- salt/common/tools/sbin/so-common | 75 +- salt/manager/tools/sbin/soup | 62 +- salt/salt/map.jinja | 2 +- salt/salt/master.defaults.yaml | 2 +- salt/salt/minion.defaults.yaml | 2 +- salt/salt/scripts/bootstrap-salt.sh | 2106 ++++++++++++++++++++++++--- setup/so-functions | 5 +- 7 files changed, 2001 insertions(+), 253 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index fc14e9d0a..87f40c9d4 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -152,15 +152,18 @@ check_salt_master_status() { return 0 } +# this is only intended to be used to check the status of the minion check_salt_minion_status() { - local timeout="${1:-5}" - echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1 - salt "$MINION_ID" test.ping -t $timeout > /dev/null 2>&1 + local minion="$1" + local timeout="${2:-5}" + local logfile="${3:-'/dev/stdout'}" + echo "Checking if the salt minion will respond to jobs" >> "$logfile" 2>&1 + salt "$minion" test.ping -t $timeout > /dev/null 2>&1 local status=$? if [ $status -gt 0 ]; then - echo " Minion did not respond" >> "$setup_log" 2>&1 + echo " Minion did not respond" >> "$logfile" 2>&1 else - echo " Received job response from salt minion" >> "$setup_log" 2>&1 + echo " Received job response from salt minion" >> "$logfile" 2>&1 fi return $status @@ -440,6 +443,24 @@ run_check_net_err() { fi } +wait_for_salt_minion() { + local minion="$1" + local timeout="${2:-5}" + local logfile="${3:-'/dev/stdout'}" + retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$logfile" 2>&1 || fail + local attempt=0 + # each attempts would take about 15 seconds + local maxAttempts=20 + until check_salt_minion_status "$minion" "$timeout" "$logfile"; do + attempt=$((attempt+1)) + if [[ $attempt -eq $maxAttempts ]]; then + return 1 + fi + sleep 10 + done + return 0 +} + salt_minion_count() { local MINIONDIR="/opt/so/saltstack/local/pillar/minions" MINIONCOUNT=$(ls -la $MINIONDIR/*.sls | grep -v adv_ | wc -l) @@ -452,19 +473,51 @@ set_os() { OS=rocky OSVER=9 is_rocky=true + is_rpm=true elif grep -q "CentOS Stream release 9" /etc/redhat-release; then OS=centos OSVER=9 is_centos=true - elif grep -q "Oracle Linux Server release 9" /etc/system-release; then - OS=oel + is_rpm=true + elif grep -q "AlmaLinux release 9" /etc/redhat-release; then + OS=alma OSVER=9 - is_oracle=true + is_alma=true + is_rpm=true + elif grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release; then + if [ -f /etc/oracle-release ]; then + OS=oracle + OSVER=9 + is_oracle=true + is_rpm=true + else + OS=rhel + OSVER=9 + is_rhel=true + is_rpm=true + fi fi cron_service_name="crond" - else - OS=ubuntu - is_ubuntu=true + elif [ -f /etc/os-release ]; then + if grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then + OSVER=focal + UBVER=20.04 + OS=ubuntu + is_ubuntu=true + is_deb=true + elif grep -q "UBUNTU_CODENAME=jammy" /etc/os-release; then + OSVER=jammy + UBVER=22.04 + OS=ubuntu + is_ubuntu=true + is_deb=true + elif grep -q "VERSION_CODENAME=bookworm" /etc/os-release; then + OSVER=bookworm + DEBVER=12 + is_debian=true + OS=debian + is_deb=true + fi cron_service_name="cron" fi } diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0666e25ae..f30c3f15d 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -467,7 +467,6 @@ stop_salt_master() { echo "" echo "Killing any queued Salt jobs on the manager." pkill -9 -ef "/usr/bin/python3 /bin/salt" >> $SOUP_LOG 2>&1 - set -e echo "" echo "Storing salt-master pid." @@ -475,6 +474,7 @@ stop_salt_master() { echo "Found salt-master PID $MASTERPID" systemctl_func "stop" "salt-master" timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option." + set -e } stop_salt_minion() { @@ -487,14 +487,12 @@ stop_salt_minion() { echo "" echo "Killing Salt jobs on this node." salt-call saltutil.kill_all_jobs --local - set -e echo "Storing salt-minion pid." MINIONPID=$(pgrep -f '/opt/saltstack/salt/bin/python3.10 /usr/bin/salt-minion' | head -1) echo "Found salt-minion PID $MINIONPID" systemctl_func "stop" "salt-minion" - set +e timeout 30 tail --pid=$MINIONPID -f /dev/null || echo "Killing salt-minion at $(date +"%T.%6N") after waiting 30s" && pkill -9 -ef /usr/bin/salt-minion set -e } @@ -633,6 +631,7 @@ upgrade_check_salt() { if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then echo "You are already running the correct version of Salt for Security Onion." else + echo "Salt needs to be upgraded to $NEWSALTVERSION." UPGRADESALT=1 fi } @@ -641,22 +640,48 @@ upgrade_salt() { SALTUPGRADED=True echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION." echo "" - # If CentOS - if [[ $OS == 'centos' ]]; then + # If rhel family + if [[ $is_rpm ]]; then echo "Removing yum versionlock for Salt." echo "" yum versionlock delete "salt-*" echo "Updating Salt packages." echo "" set +e - run_check_net_err \ - "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ - "Could not update salt, please check $SOUP_LOG for details." + # if oracle run with -r to ignore repos set by bootstrap + if [[ $OS == 'oracle' ]]; then + run_check_net_err \ + "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ + "Could not update salt, please check $SOUP_LOG for details." + # if another rhel family variant we want to run without -r to allow the bootstrap script to manage repos + else + run_check_net_err \ + "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -F -M -x python3 stable \"$NEWSALTVERSION\"" \ + "Could not update salt, please check $SOUP_LOG for details." + fi set -e echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" # Else do Ubuntu things + elif [[ $is_deb ]]; then + echo "Removing apt hold for Salt." + echo "" + apt-mark unhold "salt-common" + apt-mark unhold "salt-master" + apt-mark unhold "salt-minion" + echo "Updating Salt packages." + echo "" + set +e + run_check_net_err \ + "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -F -M -x python3 stable \"$NEWSALTVERSION\"" \ + "Could not update salt, please check $SOUP_LOG for details." + set -e + echo "Applying apt hold for Salt." + echo "" + apt-mark hold "salt-common" + apt-mark hold "salt-master" + apt-mark hold "salt-minion" fi echo "Checking if Salt was upgraded." @@ -668,7 +693,7 @@ upgrade_salt() { echo "Once the issue is resolved, run soup again." echo "Exiting." echo "" - exit 0 + exit 1 else echo "Salt upgrade success." echo "" @@ -798,7 +823,7 @@ main() { if [[ $is_airgap -eq 0 ]]; then yum clean all check_os_updates - elif [[ $OS == 'oel' ]]; then + elif [[ $OS == 'oracle' ]]; then # sync remote repo down to local if not airgap repo_sync check_os_updates @@ -815,7 +840,8 @@ main() { echo "Hotfix applied" update_version enable_highstate - salt-call state.highstate -l info queue=True + (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG" + highstate else echo "" echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION." @@ -851,6 +877,14 @@ main() { echo "Upgrading Salt" # Update the repo files so it can actually upgrade upgrade_salt + + # for Debian based distro, we need to stop salt again after upgrade output below is from bootstrap-salt + # * WARN: Not starting daemons on Debian based distributions + # is not working mostly because starting them is the default behaviour. + if [[ $is_deb ]]; then + stop_salt_minion + stop_salt_master + fi fi preupgrade_changes @@ -913,7 +947,8 @@ main() { echo "" echo "Running a highstate. This could take several minutes." set +e - salt-call state.highstate -l info queue=True + (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG" + highstate set -e stop_salt_master @@ -928,7 +963,8 @@ main() { set -e echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." - salt-call state.highstate -l info queue=True + (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG" + highstate postupgrade_changes [[ $is_airgap -eq 0 ]] && unmount_update diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 1120685fb..131ff46ca 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -23,7 +23,7 @@ {% if grains.os|lower in ['Rocky', 'redhat', 'CentOS Stream'] %} {% set UPGRADECOMMAND = 'yum clean all ; /usr/sbin/bootstrap-salt.sh -s 120 -r -F -x python3 stable ' ~ SALTVERSION %} {% elif grains.os_family|lower == 'debian' %} - {% set UPGRADECOMMAND = '/usr/sbin/bootstrap-salt.sh -s 120 -r -F -x python3 stable ' ~ SALTVERSION %} + {% set UPGRADECOMMAND = '/usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION %} {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minion version ' ~ SALTVERSION %} diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index 126039802..40b6f5268 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -2,4 +2,4 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: master: - version: 3006.1 + version: 3006.3 diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index 7e1540d17..71fd18f96 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -2,6 +2,6 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: minion: - version: 3006.1 + version: 3006.3 check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default service_start_delay: 30 # in seconds. diff --git a/salt/salt/scripts/bootstrap-salt.sh b/salt/salt/scripts/bootstrap-salt.sh index 47d25949c..a016524e6 100644 --- a/salt/salt/scripts/bootstrap-salt.sh +++ b/salt/salt/scripts/bootstrap-salt.sh @@ -14,7 +14,7 @@ # # BUGS: https://github.com/saltstack/salt-bootstrap/issues # -# COPYRIGHT: (c) 2012-2021 by the SaltStack Team, see AUTHORS.rst for more +# COPYRIGHT: (c) 2012-2022 by the SaltStack Team, see AUTHORS.rst for more # details. # # LICENSE: Apache 2.0 @@ -23,7 +23,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2021.09.17" +__ScriptVersion="2023.08.03" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -224,7 +224,6 @@ _KEEP_TEMP_FILES=${BS_KEEP_TEMP_FILES:-$BS_FALSE} _TEMP_CONFIG_DIR="null" _SALTSTACK_REPO_URL="https://github.com/saltstack/salt.git" _SALT_REPO_URL=${_SALTSTACK_REPO_URL} -_DOWNSTREAM_PKG_REPO=$BS_FALSE _TEMP_KEYS_DIR="null" _SLEEP="${__DEFAULT_SLEEP}" _INSTALL_MASTER=$BS_FALSE @@ -268,6 +267,8 @@ _CUSTOM_MASTER_CONFIG="null" _CUSTOM_MINION_CONFIG="null" _QUIET_GIT_INSTALLATION=$BS_FALSE _REPO_URL="repo.saltproject.io" +_ONEDIR_DIR="salt" +_ONEDIR_NIGHTLY_DIR="salt-dev/${_ONEDIR_DIR}" _PY_EXE="python3" _INSTALL_PY="$BS_FALSE" _TORNADO_MAX_PY3_VERSION="5.0" @@ -275,6 +276,9 @@ _POST_NEON_INSTALL=$BS_FALSE _MINIMUM_PIP_VERSION="9.0.1" _MINIMUM_SETUPTOOLS_VERSION="9.1" _POST_NEON_PIP_INSTALL_ARGS="--prefix=/usr" +_PIP_DOWNLOAD_ARGS="" +_QUICK_START="$BS_FALSE" +_AUTO_ACCEPT_MINION_KEYS="$BS_FALSE" # Defaults for install arguments ITYPE="stable" @@ -290,110 +294,130 @@ __usage() { Usage : ${__ScriptName} [options] [install-type-args] Installation types: - - stable Install latest stable release. This is the default - install type - - stable [branch] Install latest version on a branch. Only supported - for packages available at repo.saltproject.io - - stable [version] Install a specific version. Only supported for - packages available at repo.saltproject.io - To pin a 3xxx minor version, specify it as 3xxx.0 - - testing RHEL-family specific: configure EPEL testing repo - - git Install from the head of the master branch - - git [ref] Install from any git ref (such as a branch, tag, or - commit) + - stable Install latest stable release. This is the default + install type + - stable [branch] Install latest version on a branch. Only supported + for packages available at repo.saltproject.io + - stable [version] Install a specific version. Only supported for + packages available at repo.saltproject.io + To pin a 3xxx minor version, specify it as 3xxx.0 + - testing RHEL-family specific: configure EPEL testing repo + - git Install from the head of the master branch + - git [ref] Install from any git ref (such as a branch, tag, or + commit) + - onedir Install latest onedir release. + - onedir [version] Install a specific version. Only supported for + onedir packages available at repo.saltproject.io + + - onedir_rc Install latest onedir RC release. + - onedir_rc [version] Install a specific version. Only supported for + onedir RC packages available at repo.saltproject.io + - old-stable Install latest old stable release. + - old-stable [branch] Install latest version on a branch. Only supported + for packages available at repo.saltproject.io + - old-stable [version] Install a specific version. Only supported for + packages available at repo.saltproject.io + To pin a 3xxx minor version, specify it as 3xxx.0 Examples: - ${__ScriptName} - ${__ScriptName} stable - - ${__ScriptName} stable 2017.7 - - ${__ScriptName} stable 2017.7.2 + - ${__ScriptName} stable 3006 + - ${__ScriptName} stable 3006.1 - ${__ScriptName} testing - ${__ScriptName} git - ${__ScriptName} git 2017.7 - ${__ScriptName} git v2017.7.2 - ${__ScriptName} git 06f249901a2e2f1ed310d58ea3921a129f214358 + - ${__ScriptName} onedir + - ${__ScriptName} onedir 3006 + - ${__ScriptName} onedir_rc + - ${__ScriptName} onedir_rc 3006 + - ${__ScriptName} old-stable + - ${__ScriptName} old-stable 3005 + - ${__ScriptName} old-stable 3005.1 + Options: - -h Display this message - -v Display script version - -n No colours - -D Show debug output + -a Pip install all Python pkg dependencies for Salt. Requires -V to install + all pip pkgs into the virtualenv. + (Only available for Ubuntu based distributions) + -A Pass the salt-master DNS name or IP. This will be stored under + \${BS_SALT_ETC_DIR}/minion.d/99-master-address.conf + -b Assume that dependencies are already installed and software sources are + set up. If git is selected, git tree is still checked out as dependency + step. -c Temporary configuration directory - -g Salt Git repository URL. Default: ${_SALTSTACK_REPO_URL} - -w Install packages from downstream package repository rather than - upstream, saltstack package repository. This is currently only - implemented for SUSE. - -k Temporary directory holding the minion keys which will pre-seed - the master. - -s Sleep time used when waiting for daemons to start, restart and when - checking for the services running. Default: ${__DEFAULT_SLEEP} - -L Also install salt-cloud and required python-libcloud package - -M Also install salt-master - -S Also install salt-syndic - -N Do not install salt-minion - -X Do not start daemons after installation - -d Disables checking if Salt services are enabled to start on system boot. - You can also do this by touching /tmp/disable_salt_checks on the target - host. Default: \${BS_FALSE} - -P Allow pip based installations. On some distributions the required salt - packages or its dependencies are not available as a package for that - distribution. Using this flag allows the script to use pip as a last - resort method. NOTE: This only works for functions which actually - implement pip based installations. - -U If set, fully upgrade the system prior to bootstrapping Salt - -I If set, allow insecure connections while downloading any files. For - example, pass '--no-check-certificate' to 'wget' or '--insecure' to - 'curl'. On Debian and Ubuntu, using this option with -U allows obtaining - GnuPG archive keys insecurely if distro has changed release signatures. - -F Allow copied files to overwrite existing (config, init.d, etc) - -K If set, keep the temporary files in the temporary directories specified - with -c and -k -C Only run the configuration function. Implies -F (forced overwrite). To overwrite Master or Syndic configs, -M or -S, respectively, must also be specified. Salt installation will be ommitted, but some of the dependencies could be installed to write configuration with -j or -J. - -A Pass the salt-master DNS name or IP. This will be stored under - \${BS_SALT_ETC_DIR}/minion.d/99-master-address.conf - -i Pass the salt-minion id. This will be stored under - \${BS_SALT_ETC_DIR}/minion_id - -p Extra-package to install while installing Salt dependencies. One package - per -p flag. You are responsible for providing the proper package name. - -H Use the specified HTTP proxy for all download URLs (including https://). - For example: http://myproxy.example.com:3128 - -b Assume that dependencies are already installed and software sources are - set up. If git is selected, git tree is still checked out as dependency - step. + -d Disables checking if Salt services are enabled to start on system boot. + You can also do this by touching /tmp/disable_salt_checks on the target + host. Default: \${BS_FALSE} + -D Show debug output -f Force shallow cloning for git installations. This may result in an "n/a" in the version number. - -l Disable ssl checks. When passed, switches "https" calls to "http" where - possible. - -V Install Salt into virtualenv - (only available for Ubuntu based distributions) - -a Pip install all Python pkg dependencies for Salt. Requires -V to install - all pip pkgs into the virtualenv. - (Only available for Ubuntu based distributions) - -r Disable all repository configuration performed by this script. This - option assumes all necessary repository configuration is already present - on the system. - -R Specify a custom repository URL. Assumes the custom repository URL - points to a repository that mirrors Salt packages located at - repo.saltproject.io. The option passed with -R replaces the - "repo.saltproject.io". If -R is passed, -r is also set. Currently only - works on CentOS/RHEL and Debian based distributions. - -J Replace the Master config file with data passed in as a JSON string. If - a Master config file is found, a reasonable effort will be made to save - the file with a ".bak" extension. If used in conjunction with -C or -F, - no ".bak" file will be created as either of those options will force - a complete overwrite of the file. + -F Allow copied files to overwrite existing (config, init.d, etc) + -g Salt Git repository URL. Default: ${_SALTSTACK_REPO_URL} + -h Display this message + -H Use the specified HTTP proxy for all download URLs (including https://). + For example: http://myproxy.example.com:3128 + -i Pass the salt-minion id. This will be stored under + \${BS_SALT_ETC_DIR}/minion_id + -I If set, allow insecure connections while downloading any files. For + example, pass '--no-check-certificate' to 'wget' or '--insecure' to + 'curl'. On Debian and Ubuntu, using this option with -U allows obtaining + GnuPG archive keys insecurely if distro has changed release signatures. -j Replace the Minion config file with data passed in as a JSON string. If a Minion config file is found, a reasonable effort will be made to save the file with a ".bak" extension. If used in conjunction with -C or -F, no ".bak" file will be created as either of those options will force a complete overwrite of the file. + -J Replace the Master config file with data passed in as a JSON string. If + a Master config file is found, a reasonable effort will be made to save + the file with a ".bak" extension. If used in conjunction with -C or -F, + no ".bak" file will be created as either of those options will force + a complete overwrite of the file. + -k Temporary directory holding the minion keys which will pre-seed + the master. + -K If set, keep the temporary files in the temporary directories specified + with -c and -k + -l Disable ssl checks. When passed, switches "https" calls to "http" where + possible. + -L Also install salt-cloud and required python-libcloud package + -M Also install salt-master + -n No colours + -N Do not install salt-minion + -p Extra-package to install while installing Salt dependencies. One package + per -p flag. You are responsible for providing the proper package name. + -P Allow pip based installations. On some distributions the required salt + packages or its dependencies are not available as a package for that + distribution. Using this flag allows the script to use pip as a last + resort method. NOTE: This only works for functions which actually + implement pip based installations. -q Quiet salt installation from git (setup.py install -q) + -Q Quickstart, install the Salt master and the Salt minion. + And automatically accept the minion key. + -R Specify a custom repository URL. Assumes the custom repository URL + points to a repository that mirrors Salt packages located at + repo.saltproject.io. The option passed with -R replaces the + "repo.saltproject.io". If -R is passed, -r is also set. Currently only + works on CentOS/RHEL and Debian based distributions and macOS. + -s Sleep time used when waiting for daemons to start, restart and when + checking for the services running. Default: ${__DEFAULT_SLEEP} + -S Also install salt-syndic + -r Disable all repository configuration performed by this script. This + option assumes all necessary repository configuration is already present + on the system. + -U If set, fully upgrade the system prior to bootstrapping Salt + -v Display script version + -V Install Salt into virtualenv + (only available for Ubuntu based distributions) -x Changes the Python version used to install Salt. For CentOS 6 git installations python2.7 is supported. - Fedora git installation, CentOS 7, Debian 9, Ubuntu 16.04 and 18.04 support python3. + Fedora git installation, CentOS 7, Ubuntu 18.04 support python3. + -X Do not start daemons after installation -y Installs a different python version on host. Currently this has only been tested with CentOS 6 and is considered experimental. This will install the ius repo on the box if disable repo is false. This must be used in conjunction @@ -406,7 +430,7 @@ EOT } # ---------- end of function __usage ---------- -while getopts ':hvnDc:g:Gyx:wk:s:MSNXCPFUKIA:i:Lp:dH:bflV:J:j:rR:aq' opt +while getopts ':hvnDc:g:Gyx:k:s:MSNXCPFUKIA:i:Lp:dH:bflV:J:j:rR:aqQ' opt do case "${opt}" in @@ -422,7 +446,6 @@ do echowarn "No need to provide this option anymore, now it is a default behavior." ;; - w ) _DOWNSTREAM_PKG_REPO=$BS_TRUE ;; k ) _TEMP_KEYS_DIR="$OPTARG" ;; s ) _SLEEP=$OPTARG ;; M ) _INSTALL_MASTER=$BS_TRUE ;; @@ -451,6 +474,7 @@ do J ) _CUSTOM_MASTER_CONFIG=$OPTARG ;; j ) _CUSTOM_MINION_CONFIG=$OPTARG ;; q ) _QUIET_GIT_INSTALLATION=$BS_TRUE ;; + Q ) _QUICK_START=$BS_TRUE ;; x ) _PY_EXE="$OPTARG" ;; y ) _INSTALL_PY="$BS_TRUE" ;; @@ -572,7 +596,7 @@ fi echoinfo "Running version: ${__ScriptVersion}" echoinfo "Executed by: ${CALLER}" echoinfo "Command line: '${__ScriptFullName} ${__ScriptArgs}'" -#echowarn "Running the unstable version of ${__ScriptName}" +echowarn "Running the unstable version of ${__ScriptName}" # Define installation type if [ "$#" -gt 0 ];then @@ -582,11 +606,17 @@ if [ "$#" -gt 0 ];then fi # Check installation type -if [ "$(echo "$ITYPE" | grep -E '(stable|testing|git)')" = "" ]; then +if [ "$(echo "$ITYPE" | grep -E '(stable|testing|git|onedir|onedir_rc|old-stable)')" = "" ]; then echoerror "Installation type \"$ITYPE\" is not known..." exit 1 fi +# Due to our modifications to install_centos_onedir it is easiest to just lock down to only allowing stable install +if [ "$(echo "$ITYPE" | grep stable)" = "" ]; then + echoerror "This script has been modified to only support stable installation type. Installation type \"$ITYPE\" is not allowed..." + exit 1 +fi + # If doing a git install, check what branch/tag/sha will be checked out if [ "$ITYPE" = "git" ]; then if [ "$#" -eq 0 ];then @@ -602,23 +632,123 @@ if [ "$ITYPE" = "git" ]; then # If doing stable install, check if version specified elif [ "$ITYPE" = "stable" ]; then if [ "$#" -eq 0 ];then - STABLE_REV="latest" + ONEDIR_REV="latest" + _ONEDIR_REV="latest" + ITYPE="onedir" else - if [ "$(echo "$1" | grep -E '^(latest|1\.6|1\.7|2014\.1|2014\.7|2015\.5|2015\.8|2016\.3|2016\.11|2017\.7|2018\.3|2019\.2|3000|3001|3002|3003|3004)$')" != "" ]; then - STABLE_REV="$1" + if [ "$(echo "$1" | grep -E '^(nightly|latest|3005|3006)$')" != "" ]; then + ONEDIR_REV="$1" + _ONEDIR_REV="$1" + ITYPE="onedir" shift - elif [ "$(echo "$1" | grep -E '^(2[0-9]*\.[0-9]*\.[0-9]*|[3-9][0-9]{3}(\.[0-9]*)?)$')" != "" ]; then + elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[5-9](\.[0-9]*)?)')" != "" ]; then + ONEDIR_REV="minor/$1" + _ONEDIR_REV="$1" + ITYPE="onedir" + shift + else + echo "Unknown stable version: $1 (valid: 3005, 3006, latest)" + exit 1 + fi + fi + +# If doing old-stable install, check if version specified +elif [ "$ITYPE" = "old-stable" ]; then + if [ "$#" -eq 0 ];then + ITYPE="stable" + else + if [ "$(echo "$1" | grep -E '^(3003|3004|3005)$')" != "" ]; then + STABLE_REV="$1" + ITYPE="stable" + shift + elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{3}(\.[0-9]*)?)$')" != "" ]; then # Handle the 3xxx.0 version as 3xxx archive (pin to minor) and strip the fake ".0" suffix + ITYPE="stable" STABLE_REV=$(echo "$1" | sed -E 's/^([3-9][0-9]{3})\.0$/\1/') if [ "$(uname)" != "Darwin" ]; then STABLE_REV="archive/$STABLE_REV" fi shift else - echo "Unknown stable version: $1 (valid: 1.6, 1.7, 2014.1, 2014.7, 2015.5, 2015.8, 2016.3, 2016.11, 2017.7, 2018.3, 2019.2, 3000, 3001, 3002, 3003, 3004, latest, \$MAJOR.\$MINOR.\$PATCH until 2019.2, \$MAJOR or \$MAJOR.\$PATCH starting from 3000)" + echo "Unknown old stable version: $1 (valid: 3003, 3004, 3005)" exit 1 fi fi + +elif [ "$ITYPE" = "onedir" ]; then + if [ "$#" -eq 0 ];then + ONEDIR_REV="latest" + else + if [ "$(echo "$1" | grep -E '^(nightly|latest|3005|3006)$')" != "" ]; then + ONEDIR_REV="$1" + shift + elif [ "$(echo "$1" | grep -E '^(3005(\.[0-9]*)?)')" != "" ]; then + # Handle the 3005.0 version as 3005 archive (pin to minor) and strip the fake ".0" suffix + ONEDIR_REV=$(echo "$1" | sed -E 's/^(3005)\.0$/\1/') + ONEDIR_REV="minor/$ONEDIR_REV" + shift + elif [ "$(echo "$1" | grep -E '^([3-9][0-9]{3}(\.[0-9]*)?)')" != "" ]; then + ONEDIR_REV="minor/$1" + shift + else + echo "Unknown onedir version: $1 (valid: 3005, 3006, latest, nightly.)" + exit 1 + fi + fi + +elif [ "$ITYPE" = "onedir_rc" ]; then + # Change the _ONEDIR_DIR to be the location for the RC packages + _ONEDIR_DIR="salt_rc/salt" + + # Change ITYPE to onedir so we use the regular onedir functions + ITYPE="onedir" + + if [ "$#" -eq 0 ];then + ONEDIR_REV="latest" + else + if [ "$(echo "$1" | grep -E '^(latest)$')" != "" ]; then + ONEDIR_REV="$1" + shift + elif [ "$(echo "$1" | grep -E '^([3-9][0-9]{3}?rc[0-9]-[0-9]$)')" != "" ]; then + # Handle the 3xxx.0 version as 3xxx archive (pin to minor) and strip the fake ".0" suffix + #ONEDIR_REV=$(echo "$1" | sed -E 's/^([3-9][0-9]{3})\.0$/\1/') + ONEDIR_REV="minor/$1" + shift + elif [ "$(echo "$1" | grep -E '^([3-9][0-9]{3}\.[0-9]?rc[0-9]$)')" != "" ]; then + # Handle the 3xxx.0 version as 3xxx archive (pin to minor) and strip the fake ".0" suffix + #ONEDIR_REV=$(echo "$1" | sed -E 's/^([3-9][0-9]{3})\.0$/\1/') + ONEDIR_REV="minor/$1" + shift + else + echo "Unknown onedir_rc version: $1 (valid: 3005-1, latest.)" + exit 1 + fi + fi +fi + +# Doing a quick start, so install master +# set master address to 127.0.0.1 +if [ "$_QUICK_START" -eq "$BS_TRUE" ]; then + # make install type is stable + ITYPE="stable" + + # make sure the revision is latest + STABLE_REV="latest" + ONEDIR_REV="latest" + + # make sure we're installing the master + _INSTALL_MASTER=$BS_TRUE + + # override incase install minion + # is set to false + _INSTALL_MINION=$BS_TRUE + + # Set master address to loopback IP + _SALT_MASTER_ADDRESS="127.0.0.1" + + # Auto accept the minion key + # when the install is done. + _AUTO_ACCEPT_MINION_KEYS=$BS_TRUE fi # Check for any unparsed arguments. Should be an error. @@ -824,6 +954,18 @@ __fetch_verify() { return 1 } +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __check_url_exists +# DESCRIPTION: Checks if a URL exists +#---------------------------------------------------------------------------------------------------------------------- +__check_url_exists() { + _URL="$1" + if curl --output /dev/null --silent --fail "${_URL}"; then + return 0 + else + return 1 + fi +} #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __gather_hardware_info # DESCRIPTION: Discover hardware information @@ -945,7 +1087,7 @@ __strip_duplicates() { __sort_release_files() { KNOWN_RELEASE_FILES=$(echo "(arch|alpine|centos|debian|ubuntu|fedora|redhat|suse|\ mandrake|mandriva|gentoo|slackware|turbolinux|unitedlinux|void|lsb|system|\ - oracle|os)(-|_)(release|version)" | sed -E 's:[[:space:]]::g') + oracle|os|almalinux|rocky)(-|_)(release|version)" | sed -E 's:[[:space:]]::g') primary_release_files="" secondary_release_files="" # Sort know VS un-known files first @@ -959,7 +1101,7 @@ __sort_release_files() { done # Now let's sort by know files importance, max important goes last in the max_prio list - max_prio="redhat-release centos-release oracle-release fedora-release" + max_prio="redhat-release centos-release oracle-release fedora-release almalinux-release rocky-release" for entry in $max_prio; do if [ "$(echo "${primary_release_files}" | grep "$entry")" != "" ]; then primary_release_files=$(echo "${primary_release_files}" | sed -e "s:\\(.*\\)\\($entry\\)\\(.*\\):\\2 \\1 \\3:g") @@ -1028,6 +1170,8 @@ __gather_linux_system_info() { elif [ "${DISTRO_NAME}" = "Arch" ]; then DISTRO_NAME="Arch Linux" return + elif [ "${DISTRO_NAME}" = "Rocky" ]; then + DISTRO_NAME="Rocky Linux" fi rv=$(lsb_release -sr) [ "${rv}" != "" ] && DISTRO_VERSION=$(__parse_version_string "$rv") @@ -1086,6 +1230,8 @@ __gather_linux_system_info() { unitedlinux ) n="UnitedLinux" ;; void ) n="VoidLinux" ;; oracle ) n="Oracle Linux" ;; + almalinux ) n="AlmaLinux" ;; + rocky ) n="Rocky Linux" ;; system ) while read -r line; do [ "${n}x" != "systemx" ] && break @@ -1308,7 +1454,7 @@ __gather_system_info() { #---------------------------------------------------------------------------------------------------------------------- # shellcheck disable=SC2034 __ubuntu_derivatives_translation() { - UBUNTU_DERIVATIVES="(trisquel|linuxmint|linaro|elementary_os|neon)" + UBUNTU_DERIVATIVES="(trisquel|linuxmint|linaro|elementary_os|neon|pop)" # Mappings trisquel_6_ubuntu_base="12.04" linuxmint_13_ubuntu_base="12.04" @@ -1321,6 +1467,8 @@ __ubuntu_derivatives_translation() { neon_16_ubuntu_base="16.04" neon_18_ubuntu_base="18.04" neon_20_ubuntu_base="20.04" + neon_22_ubuntu_base="22.04" + pop_22_ubuntu_base="22.04" # Translate Ubuntu derivatives to their base Ubuntu version match=$(echo "$DISTRO_NAME_L" | grep -E ${UBUNTU_DERIVATIVES}) @@ -1380,9 +1528,13 @@ __check_dpkg_architecture() { if [ "$_CUSTOM_REPO_URL" != "null" ]; then warn_msg="Support for arm64 is experimental, make sure the custom repository used has the expected structure and contents." else - # Saltstack official repository does not yet have arm64 metadata, - # use amd64 repositories on arm64, since all pkgs are arch-independent - __REPO_ARCH="amd64" + # Saltstack official repository has arm64 metadata beginning with Debian 11, + # use amd64 repositories on arm64 for anything older, since all pkgs are arch-independent + if [ "$DISTRO_NAME_L" = "debian" ] && [ "$DISTRO_MAJOR_VERSION" -lt 11 ]; then + __REPO_ARCH="amd64" + else + __REPO_ARCH="arm64" + fi __REPO_ARCH_DEB="deb [signed-by=/usr/share/keyrings/salt-archive-keyring.gpg arch=$__REPO_ARCH]" warn_msg="Support for arm64 packages is experimental and might rely on architecture-independent packages from the amd64 repository." fi @@ -1462,6 +1614,9 @@ __ubuntu_codename_translation() { "21") DISTRO_CODENAME="hirsute" ;; + "22") + DISTRO_CODENAME="jammy" + ;; *) DISTRO_CODENAME="trusty" ;; @@ -1488,10 +1643,12 @@ __debian_derivatives_translation() { devuan_1_debian_base="8.0" devuan_2_debian_base="9.0" kali_1_debian_base="7.0" + kali_2021_debian_base="10.0" linuxmint_1_debian_base="8.0" raspbian_8_debian_base="8.0" raspbian_9_debian_base="9.0" raspbian_10_debian_base="10.0" + raspbian_11_debian_base="11.0" bunsenlabs_9_debian_base="9.0" turnkey_9_debian_base="9.0" @@ -1559,6 +1716,14 @@ __debian_codename_translation() { "11") DISTRO_CODENAME="bullseye" ;; + "12") + DISTRO_CODENAME="bookworm" + # FIXME - TEMPORARY + # use bullseye packages until bookworm packages are available + DISTRO_CODENAME="bullseye" + DISTRO_MAJOR_VERSION=11 + rv=11 + ;; *) DISTRO_CODENAME="stretch" ;; @@ -1590,11 +1755,13 @@ __check_end_of_life_versions() { # = 17.04, 17.10 # = 18.10 # = 19.04, 19.10 + # = 20.10 if [ "$DISTRO_MAJOR_VERSION" -lt 16 ] || \ [ "$DISTRO_MAJOR_VERSION" -eq 17 ] || \ [ "$DISTRO_MAJOR_VERSION" -eq 19 ] || \ { [ "$DISTRO_MAJOR_VERSION" -eq 16 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ - { [ "$DISTRO_MAJOR_VERSION" -eq 18 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; }; then + { [ "$DISTRO_MAJOR_VERSION" -eq 18 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ + { [ "$DISTRO_MAJOR_VERSION" -eq 20 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; }; then echoerror "End of life distributions are not supported." echoerror "Please consider upgrading to the next stable. See:" echoerror " https://wiki.ubuntu.com/Releases" @@ -1812,14 +1979,14 @@ elif [ "${DISTRO_NAME_L}" = "debian" ]; then __debian_codename_translation fi -if [ "$(echo "${DISTRO_NAME_L}" | grep -E '(debian|ubuntu|centos|gentoo|red_hat|oracle|scientific|amazon|fedora|macosx)')" = "" ] && [ "$ITYPE" = "stable" ] && [ "$STABLE_REV" != "latest" ]; then +if [ "$(echo "${DISTRO_NAME_L}" | grep -E '(debian|ubuntu|centos|gentoo|red_hat|oracle|scientific|amazon|fedora|macosx|almalinux|rocky)')" = "" ] && [ "$ITYPE" = "stable" ] && [ "$STABLE_REV" != "latest" ]; then echoerror "${DISTRO_NAME} does not have major version pegged packages support" exit 1 fi # Only RedHat based distros have testing support if [ "${ITYPE}" = "testing" ]; then - if [ "$(echo "${DISTRO_NAME_L}" | grep -E '(centos|red_hat|amazon|oracle)')" = "" ]; then + if [ "$(echo "${DISTRO_NAME_L}" | grep -E '(centos|red_hat|amazon|oracle|almalinux|rocky)')" = "" ]; then echoerror "${DISTRO_NAME} does not have testing packages support" exit 1 fi @@ -1850,10 +2017,6 @@ if [ "$ITYPE" = "git" ]; then if [ "$__NEW_VS_TAG_REGEX_MATCH" = "MATCH" ]; then _POST_NEON_INSTALL=$BS_TRUE __TAG_REGEX_MATCH="${__NEW_VS_TAG_REGEX_MATCH}" - if [ "$(echo "${GIT_REV}" | cut -c -1)" != "v" ]; then - # We do this to properly clone tags - GIT_REV="v${GIT_REV}" - fi echodebug "Post Neon Tag Regex Match On: ${GIT_REV}" else __TAG_REGEX_MATCH=$(echo "${GIT_REV}" | sed -E 's/^(v?[0-9]{1,4}\.[0-9]{1,2})(\.[0-9]{1,2})?.*$/MATCH/') @@ -1865,10 +2028,6 @@ if [ "$ITYPE" = "git" ]; then if [ "$__NEW_VS_TAG_REGEX_MATCH" = "MATCH" ]; then _POST_NEON_INSTALL=$BS_TRUE __TAG_REGEX_MATCH="${__NEW_VS_TAG_REGEX_MATCH}" - if [ "$(echo "${GIT_REV}" | cut -c -1)" != "v" ]; then - # We do this to properly clone tags - GIT_REV="v${GIT_REV}" - fi echodebug "Post Neon Tag Regex Match On: ${GIT_REV}" else __TAG_REGEX_MATCH=$(echo "${GIT_REV}" | sed 's/^.*\(v\?[[:digit:]]\{1,4\}\.[[:digit:]]\{1,2\}\)\(\.[[:digit:]]\{1,2\}\)\?.*$/MATCH/') @@ -2031,20 +2190,13 @@ __rpm_import_gpg() { #---------------------------------------------------------------------------------------------------------------------- __yum_install_noinput() { - ENABLE_EPEL_CMD="" - # Skip Amazon Linux for the first round, since EPEL is no longer required. - # See issue #724 - if [ $_DISABLE_REPOS -eq $BS_FALSE ] && [ "$DISTRO_NAME_L" != "amazon_linux_ami" ]; then - ENABLE_EPEL_CMD="--enablerepo=${_EPEL_REPO}" - fi - if [ "$DISTRO_NAME_L" = "oracle_linux" ]; then # We need to install one package at a time because --enablerepo=X disables ALL OTHER REPOS!!!! for package in "${@}"; do - yum -y install "${package}" || yum -y install "${package}" ${ENABLE_EPEL_CMD} || return $? + yum -y install "${package}" || yum -y install "${package}" || return $? done else - yum -y install "${@}" ${ENABLE_EPEL_CMD} || return $? + yum -y install "${@}" || return $? fi } # ---------- end of function __yum_install_noinput ---------- @@ -2057,6 +2209,15 @@ __dnf_install_noinput() { dnf -y install "${@}" || return $? } # ---------- end of function __dnf_install_noinput ---------- +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __tdnf_install_noinput +# DESCRIPTION: (DRY) tdnf install with noinput options +#---------------------------------------------------------------------------------------------------------------------- +__tdnf_install_noinput() { + + tdnf -y install "${@}" || return $? +} # ---------- end of function __tdnf_install_noinput ---------- + #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __git_clone_and_checkout # DESCRIPTION: (DRY) Helper function to clone and checkout salt to a @@ -2582,7 +2743,7 @@ __activate_virtualenv() { # NAME: __install_pip_pkgs # DESCRIPTION: Return 0 or 1 if successfully able to install pip packages. Can provide a different python version to # install pip packages with. If $py_ver is not specified it will use the default python version. -# PARAMETERS: pkgs, py_ver +# PARAMETERS: pkgs, py_ver, upgrade #---------------------------------------------------------------------------------------------------------------------- __install_pip_pkgs() { @@ -2751,15 +2912,15 @@ EOM fi echodebug "Running '${_pip_cmd} install wheel ${_setuptools_dep}'" - ${_pip_cmd} install ${_POST_NEON_PIP_INSTALL_ARGS} wheel "${_setuptools_dep}" + ${_pip_cmd} install --upgrade ${_POST_NEON_PIP_INSTALL_ARGS} wheel "${_setuptools_dep}" echoinfo "Installing salt using ${_py_exe}" cd "${_SALT_GIT_CHECKOUT_DIR}" || return 1 mkdir /tmp/git/deps echoinfo "Downloading Salt Dependencies from PyPi" - echodebug "Running '${_pip_cmd} download -d /tmp/git/deps .'" - ${_pip_cmd} download -d /tmp/git/deps . || (echo "Failed to download salt dependencies" && return 1) + echodebug "Running '${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} .'" + ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} . || (echo "Failed to download salt dependencies" && return 1) echoinfo "Installing Downloaded Salt Dependencies" echodebug "Running '${_pip_cmd} install --ignore-installed ${_POST_NEON_PIP_INSTALL_ARGS} /tmp/git/deps/*'" @@ -2918,7 +3079,8 @@ __enable_universe_repository() { __install_saltstack_ubuntu_repository() { # Workaround for latest non-LTS Ubuntu if { [ "$DISTRO_MAJOR_VERSION" -eq 20 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ - { [ "$DISTRO_MAJOR_VERSION" -eq 21 ] && [ "$DISTRO_MINOR_VERSION" -eq 04 ]; }; then + # remove 22 version when salt packages for 22.04 are available + [ "$DISTRO_MAJOR_VERSION" -eq 21 ] || [ "$DISTRO_MAJOR_VERSION" -eq 22 ]; then echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages for previous LTS release. You may experience problems." UBUNTU_VERSION=20.04 UBUNTU_CODENAME="focal" @@ -2957,6 +3119,58 @@ __install_saltstack_ubuntu_repository() { __wait_for_apt apt-get update || return 1 } +__install_saltstack_ubuntu_onedir_repository() { + # Workaround for latest non-LTS Ubuntu + if { [ "$DISTRO_MAJOR_VERSION" -eq 20 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ + [ "$DISTRO_MAJOR_VERSION" -eq 21 ]; then + echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages for previous LTS release. You may experience problems." + UBUNTU_VERSION=20.04 + UBUNTU_CODENAME="focal" + else + UBUNTU_VERSION=${DISTRO_VERSION} + UBUNTU_CODENAME=${DISTRO_CODENAME} + fi + + # Install downloader backend for GPG keys fetching + __PACKAGES='wget' + + # Required as it is not installed by default on Ubuntu 18+ + if [ "$DISTRO_MAJOR_VERSION" -ge 18 ]; then + __PACKAGES="${__PACKAGES} gnupg" + fi + + # Make sure https transport is available + if [ "$HTTP_VAL" = "https" ] ; then + __PACKAGES="${__PACKAGES} apt-transport-https ca-certificates" + fi + + # shellcheck disable=SC2086,SC2090 + __apt_get_install_noinput ${__PACKAGES} || return 1 + + __PY_VERSION_REPO="apt" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + __PY_VERSION_REPO="py3" + fi + + # SaltStack's stable Ubuntu repository: + SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/${ONEDIR_REV}/" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/" + fi + echo "$__REPO_ARCH_DEB $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/salt.list + + if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005)')" != "" ]; then + __apt_key_fetch "${SALTSTACK_UBUNTU_URL}salt-archive-keyring.gpg" || return 1 + elif [ "$(echo "${ONEDIR_REV}" | grep -E '(latest|nightly)')" != "" ]; then + __apt_key_fetch "${SALTSTACK_UBUNTU_URL}salt-archive-keyring.gpg" || \ + __apt_key_fetch "${SALTSTACK_UBUNTU_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 + else + __apt_key_fetch "${SALTSTACK_UBUNTU_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 + fi + + __wait_for_apt apt-get update || return 1 +} + install_ubuntu_deps() { if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then # Install add-apt-repository @@ -3032,7 +3246,7 @@ install_ubuntu_stable_deps() { if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then if [ "${_INSECURE_DL}" -eq $BS_TRUE ]; then - if [ "$DISTRO_MAJOR_VERSION" -ge 20 ] || [ "$DISTRO_MAJOR_VERSION" -ge 21 ]; then + if [ "$DISTRO_MAJOR_VERSION" -ge 20 ] || [ "$DISTRO_MAJOR_VERSION" -ge 21 ] || [ "$DISTRO_MAJOR_VERSION" -ge 22 ]; then __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && apt-get update || return 1 else __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && @@ -3113,6 +3327,9 @@ install_ubuntu_git_deps() { fi else __PACKAGES="python${PY_PKG_VER}-dev python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" + if [ "$DISTRO_MAJOR_VERSION" -ge 22 ]; then + __PACKAGES="${__PACKAGES} g++" + fi # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 fi @@ -3126,6 +3343,44 @@ install_ubuntu_git_deps() { return 0 } +install_ubuntu_onedir_deps() { + if [ "${_SLEEP}" -eq "${__DEFAULT_SLEEP}" ] && [ "$DISTRO_MAJOR_VERSION" -lt 16 ]; then + # The user did not pass a custom sleep value as an argument, let's increase the default value + echodebug "On Ubuntu systems we increase the default sleep value to 10." + echodebug "See https://github.com/saltstack/salt/issues/12248 for more info." + _SLEEP=10 + fi + + if [ $_START_DAEMONS -eq $BS_FALSE ]; then + echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." + fi + + # No user interaction, libc6 restart services for example + export DEBIAN_FRONTEND=noninteractive + + __wait_for_apt apt-get update || return 1 + + if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then + if [ "${_INSECURE_DL}" -eq $BS_TRUE ]; then + if [ "$DISTRO_MAJOR_VERSION" -ge 20 ] || [ "$DISTRO_MAJOR_VERSION" -ge 21 ]; then + __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && apt-get update || return 1 + else + __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && + apt-key update && apt-get update || return 1 + fi + fi + + __apt_get_upgrade_noinput || return 1 + fi + + if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then + __check_dpkg_architecture || return 1 + __install_saltstack_ubuntu_onedir_repository || return 1 + fi + + install_ubuntu_deps || return 1 +} + install_ubuntu_stable() { __PACKAGES="" @@ -3170,7 +3425,15 @@ install_ubuntu_git() { _POST_NEON_PIP_INSTALL_ARGS="" __install_salt_from_repo_post_neon "${_PY_EXE}" || return 1 cd "${_SALT_GIT_CHECKOUT_DIR}" || return 1 - sed -i 's:/usr/bin:/usr/local/bin:g' pkg/*.service + + # Account for new path for services files in later releases + if [ -d "pkg/common" ]; then + _SERVICE_DIR="pkg/common" + else + _SERVICE_DIR="pkg" + fi + + sed -i 's:/usr/bin:/usr/local/bin:g' ${_SERVICE_DIR}/*.service return 0 fi @@ -3185,6 +3448,28 @@ install_ubuntu_git() { return 0 } +install_ubuntu_onedir() { + __PACKAGES="" + + if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} salt-cloud" + fi + if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then + __PACKAGES="${__PACKAGES} salt-master" + fi + if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then + __PACKAGES="${__PACKAGES} salt-minion" + fi + if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then + __PACKAGES="${__PACKAGES} salt-syndic" + fi + + # shellcheck disable=SC2086 + __apt_get_install_noinput ${__PACKAGES} || return 1 + + return 0 +} + install_ubuntu_stable_post() { for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -3220,8 +3505,15 @@ install_ubuntu_git_post() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + # Account for new path for services files in later releases + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/common/salt-${fname}.service" ]; then + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg/common" + else + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg" + fi + if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" + __copyfile "${_SERVICE_DIR}/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" # Skip salt-api since the service should be opt-in and not necessarily started on boot [ $fname = "api" ] && continue @@ -3236,8 +3528,8 @@ install_ubuntu_git_post() { if [ ! -f $_upstart_conf ]; then # upstart does not know about our service, let's copy the proper file echowarn "Upstart does not appear to know about salt-$fname" - echodebug "Copying ${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-$fname.upstart to $_upstart_conf" - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.upstart" "$_upstart_conf" + echodebug "Copying ${_SERVICE_DIR}/salt-$fname.upstart to $_upstart_conf" + __copyfile "${_SERVICE_DIR}/salt-${fname}.upstart" "$_upstart_conf" # Set service to know about virtualenv if [ "${_VIRTUALENV_DIR}" != "null" ]; then echo "SALT_USE_VIRTUALENV=${_VIRTUALENV_DIR}" > /etc/default/salt-${fname} @@ -3349,17 +3641,8 @@ install_ubuntu_check_services() { # Debian Install Functions # __install_saltstack_debian_repository() { - if [ "$DISTRO_MAJOR_VERSION" -eq 11 ]; then - # Packages for Debian 11 at repo.saltproject.io are not yet available - # Set up repository for Debian 10 for Debian 11 for now until support - # is available at repo.saltproject.io for Debian 11. - echowarn "Debian 11 distribution detected, but stable packages requested. Trying packages from Debian 10. You may experience problems." - DEBIAN_RELEASE="10" - DEBIAN_CODENAME="buster" - else - DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION" - DEBIAN_CODENAME="$DISTRO_CODENAME" - fi + DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION" + DEBIAN_CODENAME="$DISTRO_CODENAME" __PY_VERSION_REPO="apt" if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then @@ -3391,6 +3674,50 @@ __install_saltstack_debian_repository() { __wait_for_apt apt-get update || return 1 } +__install_saltstack_debian_onedir_repository() { + DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION" + DEBIAN_CODENAME="$DISTRO_CODENAME" + + __PY_VERSION_REPO="apt" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + __PY_VERSION_REPO="py3" + fi + + # Install downloader backend for GPG keys fetching + __PACKAGES='wget' + + # Required as it is not installed by default on Debian 9+ + if [ "$DISTRO_MAJOR_VERSION" -ge 9 ]; then + __PACKAGES="${__PACKAGES} gnupg2" + fi + + # Make sure https transport is available + if [ "$HTTP_VAL" = "https" ] ; then + __PACKAGES="${__PACKAGES} apt-transport-https ca-certificates" + fi + + # shellcheck disable=SC2086,SC2090 + __apt_get_install_noinput ${__PACKAGES} || return 1 + + # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location + SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${ONEDIR_REV}/" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/" + fi + echo "$__REPO_ARCH_DEB $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/salt.list" + + if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005)')" != "" ]; then + __apt_key_fetch "${SALTSTACK_DEBIAN_URL}salt-archive-keyring.gpg" || return 1 + elif [ "$(echo "${ONEDIR_REV}" | grep -E '(latest|nightly)')" != "" ]; then + __apt_key_fetch "${SALTSTACK_DEBIAN_URL}salt-archive-keyring.gpg" || \ + __apt_key_fetch "${SALTSTACK_DEBIAN_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 + else + __apt_key_fetch "${SALTSTACK_DEBIAN_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 + fi + + __wait_for_apt apt-get update || return 1 +} + install_debian_deps() { if [ $_START_DAEMONS -eq $BS_FALSE ]; then echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." @@ -3444,6 +3771,59 @@ install_debian_deps() { return 0 } +install_debian_onedir_deps() { + if [ $_START_DAEMONS -eq $BS_FALSE ]; then + echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." + fi + + # No user interaction, libc6 restart services for example + export DEBIAN_FRONTEND=noninteractive + + __wait_for_apt apt-get update || return 1 + + if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then + # Try to update GPG keys first if allowed + if [ "${_INSECURE_DL}" -eq $BS_TRUE ]; then + if [ "$DISTRO_MAJOR_VERSION" -ge 10 ]; then + __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && apt-get update || return 1 + else + __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && + apt-key update && apt-get update || return 1 + fi + fi + + __apt_get_upgrade_noinput || return 1 + fi + + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + PY_PKG_VER=3 + else + PY_PKG_VER="" + fi + + # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 + __PACKAGES='procps pciutils' + + # YAML module is used for generating custom master/minion configs + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-yaml" + + # shellcheck disable=SC2086 + __apt_get_install_noinput ${__PACKAGES} || return 1 + + if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then + __check_dpkg_architecture || return 1 + __install_saltstack_debian_onedir_repository || return 1 + fi + + if [ "${_EXTRA_PACKAGES}" != "" ]; then + echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" + # shellcheck disable=SC2086 + __apt_get_install_noinput ${_EXTRA_PACKAGES} || return 1 + fi + + return 0 +} + install_debian_git_pre() { if ! __check_command_exists git; then __apt_get_install_noinput git || return 1 @@ -3692,7 +4072,15 @@ install_debian_git() { _POST_NEON_PIP_INSTALL_ARGS="" __install_salt_from_repo_post_neon "${_PY_EXE}" || return 1 cd "${_SALT_GIT_CHECKOUT_DIR}" || return 1 - sed -i 's:/usr/bin:/usr/local/bin:g' pkg/*.service + + # Account for new path for services files in later releases + if [ -d "pkg/common" ]; then + _SERVICE_DIR="pkg/common" + else + _SERVICE_DIR="pkg" + fi + + sed -i 's:/usr/bin:/usr/local/bin:g' ${_SERVICE_DIR}/*.service return 0 fi @@ -3720,6 +4108,28 @@ install_debian_9_git() { return 0 } +install_debian_onedir() { + __PACKAGES="" + + if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} salt-cloud" + fi + if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then + __PACKAGES="${__PACKAGES} salt-master" + fi + if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then + __PACKAGES="${__PACKAGES} salt-minion" + fi + if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then + __PACKAGES="${__PACKAGES} salt-syndic" + fi + + # shellcheck disable=SC2086 + __apt_get_install_noinput ${__PACKAGES} || return 1 + + return 0 +} + install_debian_git_post() { for fname in api master minion syndic; do # Skip if not meant to be installed @@ -3729,16 +4139,23 @@ install_debian_git_post() { [ "$fname" = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ "$fname" = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + # Account for new path for services files in later releases + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/common/salt-${fname}.service" ]; then + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg/common" + else + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg" + fi + # Configure SystemD for Debian 8 "Jessie" and later if [ -f /bin/systemctl ]; then if [ ! -f /lib/systemd/system/salt-${fname}.service ] || \ { [ -f /lib/systemd/system/salt-${fname}.service ] && [ $_FORCE_OVERWRITE -eq $BS_TRUE ]; }; then - if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" ]; then - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" /lib/systemd/system - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.environment" "/etc/default/salt-${fname}" + if [ -f "${_SERVICE_DIR}/salt-${fname}.service" ]; then + __copyfile "${_SERVICE_DIR}/salt-${fname}.service" /lib/systemd/system + __copyfile "${_SERVICE_DIR}/salt-${fname}.environment" "/etc/default/salt-${fname}" else # workaround before adding Debian-specific unit files to the Salt main repo - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" /lib/systemd/system + __copyfile "${_SERVICE_DIR}/salt-${fname}.service" /lib/systemd/system sed -i -e '/^Type/ s/notify/simple/' /lib/systemd/system/salt-${fname}.service fi fi @@ -3770,6 +4187,13 @@ install_debian_git_post() { done } +install_debian_2021_post() { + # Kali 2021 (debian derivative) disables all network services by default + # Using archlinux post function to enable salt systemd services + install_arch_linux_post || return 1 + return 0 +} + install_debian_restart_daemons() { [ "$_START_DAEMONS" -eq $BS_FALSE ] && return 0 @@ -3826,6 +4250,41 @@ install_debian_check_services() { # Fedora Install Functions # +__install_saltstack_fedora_onedir_repository() { + if [ "$ITYPE" = "stable" ]; then + REPO_REV="$ONEDIR_REV" + else + REPO_REV="latest" + fi + + __PY_VERSION_REPO="yum" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + __PY_VERSION_REPO="py3" + fi + + GPG_KEY="SALT-PROJECT-GPG-PUBKEY-2023.pub" + + REPO_FILE="/etc/yum.repos.d/salt.repo" + + if [ ! -s "$REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then + FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" + fi + + __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + + __rpm_import_gpg "${FETCH_URL}/${GPG_KEY}" || return 1 + + yum clean metadata || return 1 + elif [ "$REPO_REV" != "latest" ]; then + echowarn "salt.repo already exists, ignoring salt version argument." + echowarn "Use -F (forced overwrite) to install $REPO_REV." + fi + + return 0 +} + install_fedora_deps() { if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then dnf -y update || return 1 @@ -3985,6 +4444,9 @@ install_fedora_git_deps() { done else __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" + if [ "${DISTRO_VERSION}" -ge 35 ]; then + __PACKAGES="${__PACKAGES} gcc-c++" + fi # shellcheck disable=SC2086 __dnf_install_noinput ${__PACKAGES} || return 1 fi @@ -4028,7 +4490,18 @@ install_fedora_git_post() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" + # Account for new path for services files in later releases + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/common/salt-${fname}.service" ]; then + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg/common" + else + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm" + fi + __copyfile "${_SERVICE_DIR}/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" + + # Salt executables are located under `/usr/local/bin/` on Fedora 36+ + #if [ "${DISTRO_VERSION}" -ge 36 ]; then + # sed -i -e 's:/usr/bin/:/usr/local/bin/:g' /lib/systemd/system/salt-*.service + #fi # Skip salt-api since the service should be opt-in and not necessarily started on boot [ $fname = "api" ] && continue @@ -4076,6 +4549,83 @@ install_fedora_check_services() { return 0 } + +install_fedora_onedir_deps() { + + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then + yum -y update || return 1 + fi + + if [ "$_DISABLE_REPOS" -eq "$BS_TRUE" ] && [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + echowarn "Detected -r or -R option while installing Salt packages for Python 3." + echowarn "Python 3 packages for older Salt releases requires the EPEL repository to be installed." + echowarn "Installing the EPEL repository automatically is disabled when using the -r or -R options." + fi + + if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ]; then + __install_saltstack_fedora_onedir_repository || return 1 + fi + + # If -R was passed, we need to configure custom repo url with rsync-ed packages + # Which is still handled in __install_saltstack_rhel_repository. This call has + # its own check in case -r was passed without -R. + if [ "$_CUSTOM_REPO_URL" != "null" ]; then + __install_saltstack_fedora_onedir_repository || return 1 + fi + + if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then + __PACKAGES="dnf-utils chkconfig" + else + __PACKAGES="yum-utils chkconfig" + fi + + __PACKAGES="${__PACKAGES} procps" + + # shellcheck disable=SC2086 + __yum_install_noinput ${__PACKAGES} || return 1 + + if [ "${_EXTRA_PACKAGES}" != "" ]; then + echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" + # shellcheck disable=SC2086 + __yum_install_noinput ${_EXTRA_PACKAGES} || return 1 + fi + + return 0 + +} + + +install_fedora_onedir() { + STABLE_REV=$ONEDIR_REV + #install_fedora_stable || return 1 + + __PACKAGES="" + + if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} salt-cloud" + fi + if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} salt-master" + fi + if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then + __PACKAGES="${__PACKAGES} salt-minion" + fi + if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} salt-syndic" + fi + + # shellcheck disable=SC2086 + __yum_install_noinput ${__PACKAGES} || return 1 + + return 0 +} + +install_fedora_onedir_post() { + STABLE_REV=$ONEDIR_REV + install_fedora_stable_post || return 1 + + return 0 +} # # Ended Fedora Install Functions # @@ -4085,27 +4635,13 @@ install_fedora_check_services() { # # CentOS Install Functions # -__install_epel_repository() { - if [ ${_EPEL_REPOS_INSTALLED} -eq $BS_TRUE ]; then - return 0 - fi - - # Check if epel repo is already enabled and flag it accordingly - if yum repolist | grep -q "^[!]\\?${_EPEL_REPO}/"; then - _EPEL_REPOS_INSTALLED=$BS_TRUE - return 0 - fi - - # Download latest 'epel-release' package for the distro version directly - epel_repo_url="${HTTP_VAL}://dl.fedoraproject.org/pub/epel/epel-release-latest-${DISTRO_MAJOR_VERSION}.noarch.rpm" - rpm -Uvh --force "$epel_repo_url" || return 1 - - _EPEL_REPOS_INSTALLED=$BS_TRUE - - return 0 -} - __install_saltstack_rhel_repository() { + if [ "${DISTRO_MAJOR_VERSION}" -ge 9 ]; then + echoerror "Old stable repository unavailable on RH variants greater than or equal to 9" + echoerror "Use the stable install type." + exit 1 + fi + if [ "$ITYPE" = "stable" ]; then repo_rev="$STABLE_REV" else @@ -4120,7 +4656,19 @@ __install_saltstack_rhel_repository() { # Avoid using '$releasever' variable for yum. # Instead, this should work correctly on all RHEL variants. base_url="${HTTP_VAL}://${_REPO_URL}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/\$basearch/${repo_rev}/" - gpg_key="SALTSTACK-GPG-KEY.pub" + if [ "${DISTRO_MAJOR_VERSION}" -eq 7 ]; then + gpg_key="SALTSTACK-GPG-KEY.pub base/RPM-GPG-KEY-CentOS-7" + elif [ "${DISTRO_MAJOR_VERSION}" -ge 9 ]; then + gpg_key="SALTSTACK-GPG-KEY2.pub" + else + gpg_key="SALTSTACK-GPG-KEY.pub" + fi + + gpg_key_urls="" + for key in $gpg_key; do + gpg_key_urls=$(printf "${base_url}${key},%s" "$gpg_key_urls") + done + repo_file="/etc/yum.repos.d/salt.repo" if [ ! -s "$repo_file" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then @@ -4130,13 +4678,80 @@ name=SaltStack ${repo_rev} Release Channel for RHEL/CentOS \$releasever baseurl=${base_url} skip_if_unavailable=True gpgcheck=1 -gpgkey=${base_url}${gpg_key} +gpgkey=${gpg_key_urls} enabled=1 enabled_metadata=1 _eof fetch_url="${HTTP_VAL}://${_REPO_URL}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${repo_rev}/" - __rpm_import_gpg "${fetch_url}${gpg_key}" || return 1 + for key in $gpg_key; do + __rpm_import_gpg "${fetch_url}${key}" || return 1 + done + + yum clean metadata || return 1 + elif [ "$repo_rev" != "latest" ]; then + echowarn "salt.repo already exists, ignoring salt version argument." + echowarn "Use -F (forced overwrite) to install $repo_rev." + fi + + return 0 +} + +__install_saltstack_rhel_onedir_repository() { + if [ "$ITYPE" = "stable" ]; then + repo_rev="$ONEDIR_REV" + else + repo_rev="latest" + fi + + __PY_VERSION_REPO="yum" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + __PY_VERSION_REPO="py3" + fi + + # Avoid using '$releasever' variable for yum. + # Instead, this should work correctly on all RHEL variants. + base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/\$basearch/${ONEDIR_REV}/" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/\$basearch/" + fi + if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005)')" != "" ] || [ "${ONEDIR_REV}" = "nightly" ]; then + if [ "${DISTRO_MAJOR_VERSION}" -eq 9 ]; then + gpg_key="SALTSTACK-GPG-KEY2.pub" + else + gpg_key="SALTSTACK-GPG-KEY.pub" + fi + else + gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" + fi + + gpg_key_urls="" + for key in $gpg_key; do + gpg_key_urls=$(printf "${base_url}${key},%s" "$gpg_key_urls") + done + + repo_file="/etc/yum.repos.d/salt.repo" + + if [ ! -s "$repo_file" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then + cat <<_eof > "$repo_file" +[saltstack] +name=SaltStack ${repo_rev} Release Channel for RHEL/CentOS \$releasever +baseurl=${base_url} +skip_if_unavailable=True +gpgcheck=1 +gpgkey=${gpg_key_urls} +enabled=1 +enabled_metadata=1 +_eof + + fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}/" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" + fi + for key in $gpg_key; do + __rpm_import_gpg "${fetch_url}${key}" || return 1 + done + yum clean metadata || return 1 elif [ "$repo_rev" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." @@ -4158,7 +4773,6 @@ install_centos_stable_deps() { fi if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ]; then - __install_epel_repository || return 1 __install_saltstack_rhel_repository || return 1 fi @@ -4179,27 +4793,29 @@ install_centos_stable_deps() { if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then # YAML module is used for generating custom master/minion configs if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then - __PACKAGES="${__PACKAGES} python3-pyyaml" + __PACKAGES="${__PACKAGES} python3-pyyaml python3-setuptools" else __PACKAGES="${__PACKAGES} python2-pyyaml" fi elif [ "$DISTRO_MAJOR_VERSION" -eq 7 ]; then # YAML module is used for generating custom master/minion configs if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then - __PACKAGES="${__PACKAGES} python36-PyYAML" + __PACKAGES="${__PACKAGES} python36-PyYAML python36-setuptools" else __PACKAGES="${__PACKAGES} PyYAML" fi else # YAML module is used for generating custom master/minion configs if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then - __PACKAGES="${__PACKAGES} python34-PyYAML" + __PACKAGES="${__PACKAGES} python34-PyYAML python34-setuptools" else __PACKAGES="${__PACKAGES} PyYAML" fi fi fi + __PACKAGES="${__PACKAGES} procps" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4216,40 +4832,29 @@ install_centos_stable_deps() { install_centos_stable() { __PACKAGES="" - local cloud='salt-cloud' - local master='salt-master' - local minion='salt-minion' - local syndic='salt-syndic' - - if echo "$STABLE_REV" | grep -q "archive";then # point release being applied - local ver=$(echo "$STABLE_REV"|awk -F/ '{print $2}') # strip archive/ - elif echo "$STABLE_REV" | egrep -vq "archive|latest";then # latest or major version(3003, 3004, etc) being applie - local ver=$STABLE_REV - fi - - if [ ! -z $ver ]; then - cloud+="-$ver" - master+="-$ver" - minion+="-$ver" - syndic+="-$ver" - fi - if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} $cloud" + __PACKAGES="${__PACKAGES} salt-cloud" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} $master" + __PACKAGES="${__PACKAGES} salt-master" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} $minion" + __PACKAGES="${__PACKAGES} salt-minion" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} $syndic" + __PACKAGES="${__PACKAGES} salt-syndic" fi # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 + # Workaround for 3.11 broken on CentOS Stream 8.x + # Re-install Python 3.6 + _py_version=$(${_PY_EXE} -c "import sys; print('{0}.{1}'.format(*sys.version_info))") + if [ "$DISTRO_MAJOR_VERSION" -eq 8 ] && [ "${_py_version}" = "3.11" ]; then + __yum_install_noinput python3 + fi + return 0 } @@ -4285,7 +4890,14 @@ install_centos_stable_post() { } install_centos_git_deps() { - install_centos_stable_deps || return 1 + # First try stable deps then fall back to onedir deps if that one fails + # if we're installing on a Red Hat based host that doesn't have the classic + # package repos available. + # Set ONEDIR_REV to STABLE_REV in case we + # end up calling install_centos_onedir_deps + ONEDIR_REV=${STABLE_REV} + install_centos_onedir_deps || \ + return 1 if [ "$_INSECURE_DL" -eq $BS_FALSE ] && [ "${_SALT_REPO_URL%%://*}" = "https" ]; then __yum_install_noinput ca-certificates || return 1 @@ -4445,10 +5057,16 @@ install_centos_git_post() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + # Account for new path for services files in later releases + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/common/salt-${fname}.service" ]; then + _SERVICE_FILE="${_SALT_GIT_CHECKOUT_DIR}/pkg/common/salt-${fname}.service" + else + _SERVICE_FILE="${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm/salt-${fname}.service" + fi if [ -f /bin/systemctl ]; then if [ ! -f "/usr/lib/systemd/system/salt-${fname}.service" ] || \ { [ -f "/usr/lib/systemd/system/salt-${fname}.service" ] && [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; }; then - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm/salt-${fname}.service" /usr/lib/systemd/system + __copyfile "${_SERVICE_FILE}" /usr/lib/systemd/system fi SYSTEMD_RELOAD=$BS_TRUE @@ -4468,6 +5086,117 @@ install_centos_git_post() { return 0 } +install_centos_onedir_deps() { + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then + yum -y update || return 1 + fi + + if [ "$_DISABLE_REPOS" -eq "$BS_TRUE" ] && [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + echowarn "Detected -r or -R option while installing Salt packages for Python 3." + echowarn "Python 3 packages for older Salt releases requires the EPEL repository to be installed." + echowarn "Installing the EPEL repository automatically is disabled when using the -r or -R options." + fi + + if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ]; then + __install_saltstack_rhel_onedir_repository || return 1 + fi + + # If -R was passed, we need to configure custom repo url with rsync-ed packages + # Which is still handled in __install_saltstack_rhel_repository. This call has + # its own check in case -r was passed without -R. + if [ "$_CUSTOM_REPO_URL" != "null" ]; then + __install_saltstack_rhel_onedir_repository || return 1 + fi + + if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then + __PACKAGES="dnf-utils chkconfig" + else + __PACKAGES="yum-utils chkconfig" + fi + + __PACKAGES="${__PACKAGES} procps" + + # shellcheck disable=SC2086 + __yum_install_noinput ${__PACKAGES} || return 1 + + if [ "${_EXTRA_PACKAGES}" != "" ]; then + echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" + # shellcheck disable=SC2086 + __yum_install_noinput ${_EXTRA_PACKAGES} || return 1 + fi + + + return 0 +} + +# This function has been modified to allow for specific versions to be installed +# when not using the salt repo +install_centos_onedir() { + __PACKAGES="" + + local cloud='salt-cloud' + local master='salt-master' + local minion='salt-minion' + local syndic='salt-syndic' + local ver="$_ONEDIR_REV" + + if [ ! -z $ver ]; then + cloud+="-$ver" + master+="-$ver" + minion+="-$ver" + syndic+="-$ver" + fi + + if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} $cloud" + fi + if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} $master" + fi + if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then + __PACKAGES="${__PACKAGES} $minion" + fi + if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} $syndic" + fi + + # shellcheck disable=SC2086 + __yum_install_noinput ${__PACKAGES} || return 1 + + return 0 +} + +install_centos_onedir_post() { + SYSTEMD_RELOAD=$BS_FALSE + + for fname in api master minion syndic; do + # Skip salt-api since the service should be opt-in and not necessarily started on boot + [ $fname = "api" ] && continue + + # Skip if not meant to be installed + [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue + [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + + if [ -f /bin/systemctl ]; then + /bin/systemctl is-enabled salt-${fname}.service > /dev/null 2>&1 || ( + /bin/systemctl preset salt-${fname}.service > /dev/null 2>&1 && + /bin/systemctl enable salt-${fname}.service > /dev/null 2>&1 + ) + + SYSTEMD_RELOAD=$BS_TRUE + elif [ -f "/etc/init.d/salt-${fname}" ]; then + /sbin/chkconfig salt-${fname} on + fi + done + + if [ "$SYSTEMD_RELOAD" -eq $BS_TRUE ]; then + /bin/systemctl daemon-reload + fi + + return 0 +} + install_centos_restart_daemons() { [ $_START_DAEMONS -eq $BS_FALSE ] && return @@ -4567,6 +5296,11 @@ install_red_hat_linux_git_deps() { return 0 } +install_red_hat_linux_onedir_deps() { + install_centos_onedir_deps || return 1 + return 0 +} + install_red_hat_enterprise_stable_deps() { install_red_hat_linux_stable_deps || return 1 return 0 @@ -4577,6 +5311,11 @@ install_red_hat_enterprise_git_deps() { return 0 } +install_red_hat_enterprise_onedir_deps() { + install_red_hat_linux_onedir_deps || return 1 + return 0 +} + install_red_hat_enterprise_linux_stable_deps() { install_red_hat_linux_stable_deps || return 1 return 0 @@ -4587,6 +5326,11 @@ install_red_hat_enterprise_linux_git_deps() { return 0 } +install_red_hat_enterprise_linux_onedir_deps() { + install_red_hat_linux_onedir_deps || return 1 + return 0 +} + install_red_hat_enterprise_server_stable_deps() { install_red_hat_linux_stable_deps || return 1 return 0 @@ -4597,6 +5341,11 @@ install_red_hat_enterprise_server_git_deps() { return 0 } +install_red_hat_enterprise_server_onedir_deps() { + install_red_hat_linux_onedir_deps || return 1 + return 0 +} + install_red_hat_enterprise_workstation_stable_deps() { install_red_hat_linux_stable_deps || return 1 return 0 @@ -4607,6 +5356,11 @@ install_red_hat_enterprise_workstation_git_deps() { return 0 } +install_red_hat_enterprise_workstation_onedir_deps() { + install_red_hat_linux_timat_deps || return 1 + return 0 +} + install_red_hat_linux_stable() { install_centos_stable || return 1 return 0 @@ -4617,6 +5371,11 @@ install_red_hat_linux_git() { return 0 } +install_red_hat_linux_onedir() { + install_centos_onedir || return 1 + return 0 +} + install_red_hat_enterprise_stable() { install_red_hat_linux_stable || return 1 return 0 @@ -4627,6 +5386,11 @@ install_red_hat_enterprise_git() { return 0 } +install_red_hat_enterprise_onedir() { + install_red_hat_linux_onedir || return 1 + return 0 +} + install_red_hat_enterprise_linux_stable() { install_red_hat_linux_stable || return 1 return 0 @@ -4637,6 +5401,11 @@ install_red_hat_enterprise_linux_git() { return 0 } +install_red_hat_enterprise_linux_onedir() { + install_red_hat_linux_onedir || return 1 + return 0 +} + install_red_hat_enterprise_server_stable() { install_red_hat_linux_stable || return 1 return 0 @@ -4647,6 +5416,11 @@ install_red_hat_enterprise_server_git() { return 0 } +install_red_hat_enterprise_server_onedir() { + install_red_hat_linux_onedir || return 1 + return 0 +} + install_red_hat_enterprise_workstation_stable() { install_red_hat_linux_stable || return 1 return 0 @@ -4657,6 +5431,11 @@ install_red_hat_enterprise_workstation_git() { return 0 } +install_red_hat_enterprise_workstation_onedir() { + install_red_hat_linux_onedir || return 1 + return 0 +} + install_red_hat_linux_stable_post() { install_centos_stable_post || return 1 return 0 @@ -4801,6 +5580,15 @@ install_red_hat_enterprise_workstation_testing_post() { # Oracle Linux Install Functions # install_oracle_linux_stable_deps() { + # Install Oracle's EPEL. + if [ ${_EPEL_REPOS_INSTALLED} -eq $BS_FALSE ]; then + _EPEL_REPO=oracle-epel-release-el${DISTRO_MAJOR_VERSION} + if ! rpm -q "${_EPEL_REPO}" > /dev/null; then + __yum_install_noinput "${_EPEL_REPO}" + fi + _EPEL_REPOS_INSTALLED=$BS_TRUE + fi + install_centos_stable_deps || return 1 return 0 } @@ -4810,6 +5598,11 @@ install_oracle_linux_git_deps() { return 0 } +install_oracle_linux_onedir_deps() { + install_centos_onedir_deps || return 1 + return 0 +} + install_oracle_linux_testing_deps() { install_centos_testing_deps || return 1 return 0 @@ -4825,6 +5618,11 @@ install_oracle_linux_git() { return 0 } +install_oracle_linux_onedir() { + install_centos_onedir || return 1 + return 0 +} + install_oracle_linux_testing() { install_centos_testing || return 1 return 0 @@ -4840,6 +5638,11 @@ install_oracle_linux_git_post() { return 0 } +install_oracle_linux_onedir_post() { + install_centos_onedir_post || return 1 + return 0 +} + install_oracle_linux_testing_post() { install_centos_testing_post || return 1 return 0 @@ -4859,6 +5662,162 @@ install_oracle_linux_check_services() { # ####################################################################################################################### +####################################################################################################################### +# +# AlmaLinux Install Functions +# +install_almalinux_stable_deps() { + install_centos_stable_deps || return 1 + return 0 +} + +install_almalinux_git_deps() { + install_centos_git_deps || return 1 + return 0 +} + +install_almalinux_onedir_deps() { + install_centos_onedir_deps || return 1 + return 0 +} + +install_almalinux_testing_deps() { + install_centos_testing_deps || return 1 + return 0 +} + +install_almalinux_stable() { + install_centos_stable || return 1 + return 0 +} + +install_almalinux_git() { + install_centos_git || return 1 + return 0 +} + +install_almalinux_onedir() { + install_centos_onedir || return 1 + return 0 +} + +install_almalinux_testing() { + install_centos_testing || return 1 + return 0 +} + +install_almalinux_stable_post() { + install_centos_stable_post || return 1 + return 0 +} + +install_almalinux_git_post() { + install_centos_git_post || return 1 + return 0 +} + +install_almalinux_onedir_post() { + install_centos_onedir_post || return 1 + return 0 +} + +install_almalinux_testing_post() { + install_centos_testing_post || return 1 + return 0 +} + +install_almalinux_restart_daemons() { + install_centos_restart_daemons || return 1 + return 0 +} + +install_almalinux_check_services() { + install_centos_check_services || return 1 + return 0 +} +# +# Ended AlmaLinux Install Functions +# +####################################################################################################################### + +####################################################################################################################### +# +# Rocky Linux Install Functions +# +install_rocky_linux_stable_deps() { + install_centos_stable_deps || return 1 + return 0 +} + +install_rocky_linux_git_deps() { + install_centos_git_deps || return 1 + return 0 +} + +install_rocky_linux_onedir_deps() { + install_centos_onedir_deps || return 1 + return 0 +} + +install_rocky_linux_testing_deps() { + install_centos_testing_deps || return 1 + return 0 +} + +install_rocky_linux_stable() { + install_centos_stable || return 1 + return 0 +} + +install_rocky_linux_onedir() { + install_centos_onedir || return 1 + return 0 +} + +install_rocky_linux_git() { + install_centos_git || return 1 + return 0 +} + +install_rocky_linux_testing() { + install_centos_testing || return 1 + return 0 +} + +install_rocky_linux_stable_post() { + install_centos_stable_post || return 1 + return 0 +} + +install_rocky_linux_git_post() { + install_centos_git_post || return 1 + return 0 +} + +install_rocky_linux_onedir_post() { + install_centos_onedir_post || return 1 + return 0 +} + +install_rocky_linux_testing_post() { + install_centos_testing_post || return 1 + return 0 +} + +install_rocky_linux_restart_daemons() { + install_centos_restart_daemons || return 1 + return 0 +} + +install_rocky_linux_check_services() { + install_centos_check_services || return 1 + return 0 +} +# +# Ended Rocky Linux Install Functions +# +####################################################################################################################### + ####################################################################################################################### # # Scientific Linux Install Functions @@ -4873,6 +5832,11 @@ install_scientific_linux_git_deps() { return 0 } +install_scientific_linux_onedir_deps() { + install_centos_onedir_deps || return 1 + return 0 +} + install_scientific_linux_testing_deps() { install_centos_testing_deps || return 1 return 0 @@ -4888,6 +5852,11 @@ install_scientific_linux_git() { return 0 } +install_scientific_linux_onedir() { + install_centos_onedir || return 1 + return 0 +} + install_scientific_linux_testing() { install_centos_testing || return 1 return 0 @@ -4903,6 +5872,11 @@ install_scientific_linux_git_post() { return 0 } +install_scientific_linux_onedir_post() { + install_centos_onedir_post || return 1 + return 0 +} + install_scientific_linux_testing_post() { install_centos_testing_post || return 1 return 0 @@ -4936,6 +5910,11 @@ install_cloud_linux_git_deps() { return 0 } +install_cloud_linux_onedir_deps() { + install_centos_onedir_deps || return 1 + return 0 +} + install_cloud_linux_testing_deps() { install_centos_testing_deps || return 1 return 0 @@ -5029,8 +6008,8 @@ install_alpine_linux_git_deps() { fi fi else - apk -U add python2 py2-pip py2-setuptools || return 1 - _PY_EXE=python2 + apk -U add python3 python3-dev py3-pip py3-setuptools g++ linux-headers zeromq-dev openrc || return 1 + _PY_EXE=python3 return 0 fi @@ -5500,6 +6479,100 @@ _eof fi } +install_amazon_linux_ami_2_onedir_deps() { + # Shim to figure out if we're using old (rhel) or new (aws) rpms. + _USEAWS=$BS_FALSE + pkg_append="python" + + if [ "$ITYPE" = "onedir" ]; then + repo_rev="$ONEDIR_REV" + else + repo_rev="latest" + fi + + if echo $repo_rev | grep -E -q '^archive'; then + year=$(echo "$repo_rev" | cut -d '/' -f 2 | cut -c1-4) + else + year=$(echo "$repo_rev" | cut -c1-4) + fi + + # We need to install yum-utils before doing anything else when installing on + # Amazon Linux ECS-optimized images. See issue #974. + __yum_install_noinput yum-utils + + # Do upgrade early + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then + yum -y update || return 1 + fi + + if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then + __REPO_FILENAME="salt.repo" + __PY_VERSION_REPO="yum" + PY_PKG_VER="" + repo_label="saltstack-repo" + repo_name="SaltStack repo for Amazon Linux 2" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + __REPO_FILENAME="salt.repo" + __PY_VERSION_REPO="py3" + PY_PKG_VER=3 + repo_label="saltstack-py3-repo" + repo_name="SaltStack Python 3 repo for Amazon Linux 2" + fi + + base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/amazon/2/\$basearch/$repo_rev/" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/amazon/2/\$basearch/" + fi + + if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005)')" != "" ] || [ "${ONEDIR_REV}" = "nightly" ]; then + gpg_key="${base_url}SALTSTACK-GPG-KEY.pub,${base_url}base/RPM-GPG-KEY-CentOS-7" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + gpg_key="${base_url}SALTSTACK-GPG-KEY.pub" + fi + else + gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub" + fi + + # This should prob be refactored to use __install_saltstack_rhel_repository() + # With args passed in to do the right thing. Reformatted to be more like the + # amazon linux yum file. + if [ ! -s "/etc/yum.repos.d/${__REPO_FILENAME}" ]; then + cat <<_eof > "/etc/yum.repos.d/${__REPO_FILENAME}" +[$repo_label] +name=$repo_name +failovermethod=priority +priority=10 +gpgcheck=1 +gpgkey=$gpg_key +baseurl=$base_url +_eof + fi + + fi + + if [ "${_POST_NEON_INSTALL}" -eq $BS_FALSE ]; then + # Package python-ordereddict-1.1-2.el6.noarch is obsoleted by python26-2.6.9-2.88.amzn1.x86_64 + # which is already installed + if [ -n "${PY_PKG_VER}" ] && [ "${PY_PKG_VER}" -eq 3 ]; then + __PACKAGES="${pkg_append}${PY_PKG_VER}-m2crypto ${pkg_append}${PY_PKG_VER}-pyyaml" + else + __PACKAGES="m2crypto PyYAML ${pkg_append}-futures" + fi + + __PACKAGES="${__PACKAGES} ${pkg_append}${PY_PKG_VER}-crypto ${pkg_append}${PY_PKG_VER}-jinja2 procps-ng" + __PACKAGES="${__PACKAGES} ${pkg_append}${PY_PKG_VER}-msgpack ${pkg_append}${PY_PKG_VER}-requests ${pkg_append}${PY_PKG_VER}-zmq" + + # shellcheck disable=SC2086 + __yum_install_noinput ${__PACKAGES} || return 1 + fi + + if [ "${_EXTRA_PACKAGES}" != "" ]; then + echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" + # shellcheck disable=SC2086 + __yum_install_noinput ${_EXTRA_PACKAGES} || return 1 + fi +} + install_amazon_linux_ami_stable() { install_centos_stable || return 1 return 0 @@ -5575,6 +6648,16 @@ install_amazon_linux_ami_2_check_services() { return 0 } +install_amazon_linux_ami_2_onedir() { + install_centos_stable || return 1 + return 0 +} + +install_amazon_linux_ami_2_onedir_post() { + install_centos_stable_post || return 1 + return 0 +} + # # Ended Amazon Linux AMI Install Functions # @@ -5666,6 +6749,10 @@ install_arch_linux_git_deps() { return 0 } +install_arch_linux_onedir_deps() { + install_arch_linux_stable_deps || return 1 +} + install_arch_linux_stable() { # Pacman does not resolve dependencies on outdated versions # They always need to be updated @@ -5684,6 +6771,8 @@ install_arch_linux_stable() { install_arch_linux_git() { + _POST_NEON_PIP_INSTALL_ARGS="${_POST_NEON_PIP_INSTALL_ARGS} --use-pep517" + _PIP_DOWNLOAD_ARGS="${_PIP_DOWNLOAD_ARGS} --use-pep517" if [ "${_POST_NEON_INSTALL}" -eq $BS_TRUE ]; then __install_salt_from_repo_post_neon "${_PY_EXE}" || return 1 return 0 @@ -5741,8 +6830,15 @@ install_arch_linux_git_post() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + # Account for new path for services files in later releases + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/common/salt-${fname}.service" ]; then + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg/common" + else + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm" + fi + if [ -f /usr/bin/systemctl ]; then - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" + __copyfile "${_SERVICE_DIR}/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" # Skip salt-api since the service should be opt-in and not necessarily started on boot [ $fname = "api" ] && continue @@ -5809,11 +6905,344 @@ install_arch_check_services() { return 0 } + +install_arch_linux_onedir() { + install_arch_linux_stable || return 1 + + return 0 +} + +install_arch_linux_onedir_post() { + install_arch_linux_post || return 1 + + return 0 +} # # Ended Arch Install Functions # ####################################################################################################################### +####################################################################################################################### +# +# Photon OS Install Functions +# + +__install_saltstack_photon_onedir_repository() { + if [ "$ITYPE" = "stable" ]; then + REPO_REV="$ONEDIR_REV" + else + REPO_REV="latest" + fi + + __PY_VERSION_REPO="yum" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + __PY_VERSION_REPO="py3" + fi + + REPO_FILE="/etc/yum.repos.d/salt.repo" + + if [ ! -s "$REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then + FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" + fi + + __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + + GPG_KEY="SALT-PROJECT-GPG-PUBKEY-2023.pub" + + __rpm_import_gpg "${FETCH_URL}/${GPG_KEY}" || return 1 + + tdnf makecache || return 1 + elif [ "$REPO_REV" != "latest" ]; then + echowarn "salt.repo already exists, ignoring salt version argument." + echowarn "Use -F (forced overwrite) to install $REPO_REV." + fi + + return 0 +} + +install_photon_deps() { + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then + tdnf -y update || return 1 + fi + + __PACKAGES="${__PACKAGES:=}" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -lt 3 ]; then + echoerror "There are no Python 2 stable packages for Fedora, only Py3 packages" + return 1 + fi + + PY_PKG_VER=3 + + __PACKAGES="${__PACKAGES} libyaml procps-ng python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests python${PY_PKG_VER}-zmq" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-pip python${PY_PKG_VER}-m2crypto python${PY_PKG_VER}-pyyaml" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd" + if [ "${_EXTRA_PACKAGES}" != "" ]; then + echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" + fi + + # shellcheck disable=SC2086 + __tdnf_install_noinput ${__PACKAGES} ${_EXTRA_PACKAGES} || return 1 + + return 0 +} + +install_photon_stable_post() { + for fname in api master minion syndic; do + # Skip salt-api since the service should be opt-in and not necessarily started on boot + [ $fname = "api" ] && continue + + # Skip if not meant to be installed + [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue + [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + + systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) + sleep 1 + systemctl daemon-reload + done +} + +install_photon_git_deps() { + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + # Packages are named python3- + PY_PKG_VER=3 + else + PY_PKG_VER=2 + fi + + __PACKAGES="" + if ! __check_command_exists ps; then + __PACKAGES="${__PACKAGES} procps-ng" + fi + if ! __check_command_exists git; then + __PACKAGES="${__PACKAGES} git" + fi + + if [ -n "${__PACKAGES}" ]; then + # shellcheck disable=SC2086 + __tdnf_install_noinput ${__PACKAGES} || return 1 + __PACKAGES="" + fi + + __git_clone_and_checkout || return 1 + + if [ "${_POST_NEON_INSTALL}" -eq $BS_FALSE ]; then + + if [ "$_INSECURE_DL" -eq $BS_FALSE ] && [ "${_SALT_REPO_URL%%://*}" = "https" ]; then + __PACKAGES="${__PACKAGES} ca-certificates" + fi + if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-libcloud python${PY_PKG_VER}-netaddr" + fi + + install_photon_deps || return 1 + + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + if __check_command_exists python3; then + __python="python3" + fi + elif [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 2 ]; then + if __check_command_exists python2; then + __python="python2" + fi + else + if ! __check_command_exists python; then + echoerror "Unable to find a python binary?!" + return 1 + fi + # Let's hope it's the right one + __python="python" + fi + + grep tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" | while IFS=' + ' read -r dep; do + echodebug "Running '${__python}' -m pip install '${dep}'" + "${__python}" -m pip install "${dep}" || return 1 + done + else + __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc glibc-devel linux-devel.x86_64" + # shellcheck disable=SC2086 + __tdnf_install_noinput ${__PACKAGES} || return 1 + fi + + if [ "${DISTRO_MAJOR_VERSION}" -gt 3 ]; then + # Need newer version of setuptools on Photon + _setuptools_dep="setuptools>=${_MINIMUM_SETUPTOOLS_VERSION}" + echodebug "Running '${_PY_EXE} -m pip --upgrade install ${_setuptools_dep}'" + ${_PY_EXE} -m pip install --upgrade "${_setuptools_dep}" + fi + + # Let's trigger config_salt() + if [ "$_TEMP_CONFIG_DIR" = "null" ]; then + _TEMP_CONFIG_DIR="${_SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_photon_git() { + if [ "${_PY_EXE}" != "" ]; then + _PYEXE=${_PY_EXE} + echoinfo "Using the following python version: ${_PY_EXE} to install salt" + else + _PYEXE='python2' + fi + + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/salt/syspaths.py" ]; then + ${_PYEXE} setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install --prefix=/usr || return 1 + else + ${_PYEXE} setup.py ${SETUP_PY_INSTALL_ARGS} install --prefix=/usr || return 1 + fi + return 0 +} + +install_photon_git_post() { + for fname in api master minion syndic; do + # Skip if not meant to be installed + [ $fname = "api" ] && \ + ([ "$_INSTALL_MASTER" -eq $BS_FALSE ] || ! __check_command_exists "salt-${fname}") && continue + [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue + [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + + # Account for new path for services files in later releases + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/common/salt-${fname}.service" ]; then + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg/common" + else + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm" + fi + __copyfile "${_SERVICE_DIR}/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" + + # Salt executables are located under `/usr/local/bin/` on Fedora 36+ + #if [ "${DISTRO_VERSION}" -ge 36 ]; then + # sed -i -e 's:/usr/bin/:/usr/local/bin/:g' /lib/systemd/system/salt-*.service + #fi + + # Skip salt-api since the service should be opt-in and not necessarily started on boot + [ $fname = "api" ] && continue + + systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) + sleep 1 + systemctl daemon-reload + done +} + +install_photon_restart_daemons() { + [ $_START_DAEMONS -eq $BS_FALSE ] && return + + for fname in api master minion syndic; do + # Skip salt-api since the service should be opt-in and not necessarily started on boot + [ $fname = "api" ] && continue + + # Skip if not meant to be installed + [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue + [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + + systemctl stop salt-$fname > /dev/null 2>&1 + systemctl start salt-$fname.service && continue + echodebug "Failed to start salt-$fname using systemd" + if [ "$_ECHO_DEBUG" -eq $BS_TRUE ]; then + systemctl status salt-$fname.service + journalctl -xe + fi + done +} + +install_photon_check_services() { + for fname in api master minion syndic; do + # Skip salt-api since the service should be opt-in and not necessarily started on boot + [ $fname = "api" ] && continue + + # Skip if not meant to be installed + [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue + [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + + __check_services_systemd salt-$fname || return 1 + done + + return 0 +} + +install_photon_onedir_deps() { + + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then + tdnf -y update || return 1 + fi + + if [ "$_DISABLE_REPOS" -eq "$BS_TRUE" ] && [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + echowarn "Detected -r or -R option while installing Salt packages for Python 3." + echowarn "Python 3 packages for older Salt releases requires the EPEL repository to be installed." + echowarn "Installing the EPEL repository automatically is disabled when using the -r or -R options." + fi + + if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ]; then + __install_saltstack_photon_onedir_repository || return 1 + fi + + # If -R was passed, we need to configure custom repo url with rsync-ed packages + # Which is still handled in __install_saltstack_rhel_repository. This call has + # its own check in case -r was passed without -R. + if [ "$_CUSTOM_REPO_URL" != "null" ]; then + __install_saltstack_photon_onedir_repository || return 1 + fi + + __PACKAGES="procps-ng" + + # shellcheck disable=SC2086 + __tdnf_install_noinput ${__PACKAGES} || return 1 + + if [ "${_EXTRA_PACKAGES}" != "" ]; then + echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" + # shellcheck disable=SC2086 + __tdnf_install_noinput ${_EXTRA_PACKAGES} || return 1 + fi + + return 0 + +} + + +install_photon_onedir() { + STABLE_REV=$ONEDIR_REV + + __PACKAGES="" + + if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} salt-cloud" + fi + if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} salt-master" + fi + if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then + __PACKAGES="${__PACKAGES} salt-minion" + fi + if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} salt-syndic" + fi + + # shellcheck disable=SC2086 + __tdnf_install_noinput ${__PACKAGES} || return 1 + + return 0 +} + +install_photon_onedir_post() { + STABLE_REV=$ONEDIR_REV + install_photon_stable_post || return 1 + + return 0 +} +# +# Ended Fedora Install Functions +# +####################################################################################################################### + ####################################################################################################################### # # FreeBSD Install Functions @@ -5841,15 +7270,15 @@ install_freebsd_git_deps() { if [ "${_POST_NEON_INSTALL}" -eq $BS_FALSE ]; then - SALT_DEPENDENCIES=$(/usr/local/sbin/pkg rquery %dn py38-salt) + SALT_DEPENDENCIES=$(/usr/local/sbin/pkg rquery %dn py39-salt) # shellcheck disable=SC2086 /usr/local/sbin/pkg install -y ${SALT_DEPENDENCIES} python || return 1 - /usr/local/sbin/pkg install -y py38-requests || return 1 - /usr/local/sbin/pkg install -y py38-tornado4 || return 1 + /usr/local/sbin/pkg install -y py39-requests || return 1 + /usr/local/sbin/pkg install -y py39-tornado4 || return 1 else - /usr/local/sbin/pkg install -y python py38-pip py38-setuptools libzmq4 libunwind || return 1 + /usr/local/sbin/pkg install -y python py39-pip py39-setuptools libzmq4 libunwind || return 1 fi echodebug "Adapting paths to FreeBSD" @@ -5895,7 +7324,7 @@ install_freebsd_stable() { # installing latest version of salt from FreeBSD CURRENT ports repo # # shellcheck disable=SC2086 - /usr/local/sbin/pkg install -y py38-salt || return 1 + /usr/local/sbin/pkg install -y py39-salt || return 1 return 0 } @@ -5987,6 +7416,15 @@ install_freebsd_restart_daemons() { service salt_$fname start done } + +install_freebsd_onedir() { +# +# call install_freebsd_stable +# + install_freebsd_stable || return 1 + + return 0 +} # # Ended FreeBSD Install Functions # @@ -6021,7 +7459,7 @@ install_openbsd_git_deps() { __git_clone_and_checkout || return 1 if [ "${_POST_NEON_INSTALL}" -eq $BS_TRUE ]; then - pkg_add -I -v py-pip py-setuptools + pkg_add -I -v py3-pip py3-setuptools fi # @@ -6105,6 +7543,14 @@ install_openbsd_restart_daemons() { return 0 } +install_openbsd_onedir() { +# +# Call install_openbsd_stable +# + install_openbsd_stable || return 1 + + return 0 +} # # Ended OpenBSD Install Functions # @@ -6305,6 +7751,14 @@ install_smartos_restart_daemons() { return 0 } +install_smartos_onedir() { +# +# call install_smartos_stable +# + install_smartos_stable || return 1 + + return 0 +} # # Ended SmartOS Install Functions # @@ -6321,19 +7775,16 @@ __set_suse_pkg_repo() { # Set distro repo variable if [ "${DISTRO_MAJOR_VERSION}" -gt 2015 ]; then DISTRO_REPO="openSUSE_Tumbleweed" + elif [ "${DISTRO_MAJOR_VERSION}" -eq 15 ] && [ "${DISTRO_MINOR_VERSION}" -ge 4 ]; then + DISTRO_REPO="${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" elif [ "${DISTRO_MAJOR_VERSION}" -ge 42 ] || [ "${DISTRO_MAJOR_VERSION}" -eq 15 ]; then DISTRO_REPO="openSUSE_Leap_${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" else DISTRO_REPO="SLE_${DISTRO_MAJOR_VERSION}_SP${SUSE_PATCHLEVEL}" fi - if [ "$_DOWNSTREAM_PKG_REPO" -eq $BS_TRUE ]; then - suse_pkg_url_base="https://download.opensuse.org/repositories/systemsmanagement:/saltstack" - suse_pkg_url_path="${DISTRO_REPO}/systemsmanagement:saltstack.repo" - else - suse_pkg_url_base="${HTTP_VAL}://repo.saltproject.io/opensuse" - suse_pkg_url_path="${DISTRO_REPO}/systemsmanagement:saltstack:products.repo" - fi + suse_pkg_url_base="https://download.opensuse.org/repositories/systemsmanagement:/saltstack" + suse_pkg_url_path="${DISTRO_REPO}/systemsmanagement:saltstack.repo" SUSE_PKG_URL="$suse_pkg_url_base/$suse_pkg_url_path" } @@ -6353,7 +7804,7 @@ __version_lte() { zypper --non-interactive install --auto-agree-with-licenses python || return 1 fi - if [ "$(python -c 'import sys; V1=tuple([int(i) for i in sys.argv[1].split(".")]); V2=tuple([int(i) for i in sys.argv[2].split(".")]); print V1<=V2' "$1" "$2")" = "True" ]; then + if [ "$(${_PY_EXE} -c 'import sys; V1=tuple([int(i) for i in sys.argv[1].split(".")]); V2=tuple([int(i) for i in sys.argv[2].split(".")]); print(V1<=V2)' "$1" "$2")" = "True" ]; then __ZYPPER_REQUIRES_REPLACE_FILES=${BS_TRUE} else __ZYPPER_REQUIRES_REPLACE_FILES=${BS_FALSE} @@ -6470,7 +7921,7 @@ install_opensuse_git_deps() { fi # Check for Tumbleweed elif [ "${DISTRO_MAJOR_VERSION}" -ge 20210101 ]; then - __PACKAGES="python3-pip" + __PACKAGES="python3-pip gcc-c++ python3-pyzmq-devel" else __PACKAGES="python-pip python-setuptools gcc" fi @@ -6487,6 +7938,10 @@ install_opensuse_git_deps() { return 0 } +install_opensuse_onedir_deps() { + install_opensuse_stable_deps || return 1 +} + install_opensuse_stable() { __PACKAGES="" @@ -6519,6 +7974,10 @@ install_opensuse_git() { return 0 } +install_opensuse_onedir() { + install_opensuse_stable || return 1 +} + install_opensuse_stable_post() { for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -6563,10 +8022,17 @@ install_opensuse_git_post() { use_usr_lib=$BS_TRUE fi - if [ "${use_usr_lib}" -eq $BS_TRUE ]; then - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" "/usr/lib/systemd/system/salt-${fname}.service" + # Account for new path for services files in later releases + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/common/salt-${fname}.service" ]; then + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg/common" else - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg/" + fi + + if [ "${use_usr_lib}" -eq $BS_TRUE ]; then + __copyfile "${_SERVICE_DIR}/salt-${fname}.service" "/usr/lib/systemd/system/salt-${fname}.service" + else + __copyfile "${_SERVICE_DIR}/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" fi continue @@ -6581,6 +8047,10 @@ install_opensuse_git_post() { return 0 } +install_opensuse_onedir_post() { + install_opensuse_stable_post || return 1 +} + install_opensuse_restart_daemons() { [ $_START_DAEMONS -eq $BS_FALSE ] && return @@ -6740,6 +8210,11 @@ install_opensuse_15_git() { return 0 } +install_opensuse_15_onedir_deps() { + __opensuse_prep_install || return 1 + return 0 +} + # # End of openSUSE Leap 15 # @@ -6769,6 +8244,13 @@ install_suse_15_git_deps() { return 0 } +install_suse_15_onedir_deps() { + __opensuse_prep_install || return 1 + install_opensuse_15_onedir_deps || return 1 + + return 0 +} + install_suse_15_stable() { install_opensuse_stable || return 1 return 0 @@ -6779,6 +8261,11 @@ install_suse_15_git() { return 0 } +install_suse_15_onedir() { + install_opensuse_stable || return 1 + return 0 +} + install_suse_15_stable_post() { install_opensuse_stable_post || return 1 return 0 @@ -6789,6 +8276,11 @@ install_suse_15_git_post() { return 0 } +install_suse_15_onedir_post() { + install_opensuse_stable_post || return 1 + return 0 +} + install_suse_15_restart_daemons() { install_opensuse_restart_daemons || return 1 return 0 @@ -6871,6 +8363,11 @@ install_suse_12_git_deps() { return 0 } +install_suse_12_onedir_deps() { + install_suse_12_stable_deps || return 1 + return 0 +} + install_suse_12_stable() { install_opensuse_stable || return 1 return 0 @@ -6881,6 +8378,11 @@ install_suse_12_git() { return 0 } +install_suse_12_onedir() { + install_opensuse_stable || return 1 + return 0 +} + install_suse_12_stable_post() { install_opensuse_stable_post || return 1 return 0 @@ -6891,6 +8393,11 @@ install_suse_12_git_post() { return 0 } +install_suse_12_onedir_post() { + install_opensuse_stable_post || return 1 + return 0 +} + install_suse_12_restart_daemons() { install_opensuse_restart_daemons || return 1 return 0 @@ -6967,6 +8474,11 @@ install_suse_11_git_deps() { return 0 } +install_suse_11_onedir_deps() { + install_suse_11_stable_deps || return 1 + return 0 +} + install_suse_11_stable() { install_opensuse_stable || return 1 return 0 @@ -6977,6 +8489,11 @@ install_suse_11_git() { return 0 } +install_suse_11_onedir() { + install_opensuse_stable || return 1 + return 0 +} + install_suse_11_stable_post() { install_opensuse_stable_post || return 1 return 0 @@ -6987,6 +8504,11 @@ install_suse_11_git_post() { return 0 } +install_suse_11_onedir_post() { + install_opensuse_stable_post || return 1 + return 0 +} + install_suse_11_restart_daemons() { install_opensuse_restart_daemons || return 1 return 0 @@ -7086,11 +8608,6 @@ __gentoo_pre_dep() { mkdir /etc/portage fi - # Enable Python 3.6 target for pre Neon Salt release - if echo "${STABLE_REV}" | grep -q "2019" || [ "${ITYPE}" = "git" ] && [ "${_POST_NEON_INSTALL}" -eq $BS_FALSE ]; then - EXTRA_PYTHON_TARGET=python3_6 - fi - # Enable Python 3.7 target for Salt Neon using GIT if [ "${ITYPE}" = "git" ] && [ "${GIT_REV}" = "v3000" ]; then EXTRA_PYTHON_TARGET=python3_7 @@ -7186,6 +8703,9 @@ install_gentoo_git_deps() { __emerge ${GENTOO_GIT_PACKAGES} || return 1 fi + echoinfo "Running emerge -v1 setuptools" + __emerge -v1 setuptools || return 1 + __git_clone_and_checkout || return 1 __gentoo_post_dep || return 1 } @@ -7233,6 +8753,11 @@ install_gentoo_git() { return 0 } +install_gentoo_onedir() { + STABLE_REV=${ONEDIR_REV} + install_gentoo_stable || return 1 +} + install_gentoo_post() { for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -7268,8 +8793,15 @@ install_gentoo_git_post() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + # Account for new path for services files in later releases + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/common/salt-${fname}.service" ]; then + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg/common" + else + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg" + fi + if __check_command_exists systemctl ; then - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" + __copyfile "${_SERVICE_DIR}/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" # Skip salt-api since the service should be opt-in and not necessarily started on boot [ $fname = "api" ] && continue @@ -7315,6 +8847,10 @@ _eof return 0 } +install_gentoo_onedir_post() { + install_gentoo_post || return 1 +} + install_gentoo_restart_daemons() { [ $_START_DAEMONS -eq $BS_FALSE ] && return @@ -7466,7 +9002,46 @@ __macosx_get_packagesite() { fi PKG="salt-${STABLE_REV}-${__PY_VERSION_REPO}-${DARWIN_ARCH}.pkg" - SALTPKGCONFURL="https://repo.saltproject.io/osx/${PKG}" + SALTPKGCONFURL="https://${_REPO_URL}/osx/${PKG}" +} + +__parse_repo_json_python() { + + # Using latest, grab the right + # version from the repo.json + _JSON_VERSION=$(python - <<-EOF +import json, urllib.request +url = "https://repo.saltproject.io/salt/py3/macos/repo.json" +response = urllib.request.urlopen(url) +data = json.loads(response.read()) +version = data["${_ONEDIR_REV}"][list(data["${_ONEDIR_REV}"])[0]]['version'] +print(version) +EOF +) +echo "${_JSON_VERSION}" +} + +__macosx_get_packagesite_onedir() { + DARWIN_ARCH="x86_64" + + __PY_VERSION_REPO="py2" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + __PY_VERSION_REPO="py3" + fi + + if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then + _PKG_VERSION=$(__parse_repo_json_python) + elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*))')" != "" ]; then + _PKG_VERSION=$_ONEDIR_REV + else + _PKG_VERSION=$(__parse_repo_json_python) + fi + if [ "$(echo "$_ONEDIR_REV" | grep -E '^(3005)')" != "" ]; then + PKG="salt-${_PKG_VERSION}-macos-${DARWIN_ARCH}.pkg" + else + PKG="salt-${_PKG_VERSION}-${__PY_VERSION_REPO}-${DARWIN_ARCH}.pkg" + fi + SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/macos/${ONEDIR_REV}/${PKG}" } # Using a separate conf step to head for idempotent install... @@ -7475,11 +9050,21 @@ __configure_macosx_pkg_details() { return 0 } +__configure_macosx_pkg_details_onedir() { + __macosx_get_packagesite_onedir || return 1 + return 0 +} + install_macosx_stable_deps() { __configure_macosx_pkg_details || return 1 return 0 } +install_macosx_onedir_deps() { + __configure_macosx_pkg_details_onedir || return 1 + return 0 +} + install_macosx_git_deps() { install_macosx_stable_deps || return 1 @@ -7526,6 +9111,16 @@ install_macosx_stable() { return 0 } +install_macosx_onedir() { + install_macosx_onedir_deps || return 1 + + __fetch_url "/tmp/${PKG}" "${SALTPKGCONFURL}" || return 1 + + /usr/sbin/installer -pkg "/tmp/${PKG}" -target / || return 1 + + return 0 +} + install_macosx_git() { if [ -n "$_PY_EXE" ]; then @@ -7563,6 +9158,11 @@ install_macosx_stable_post() { return 0 } +install_macosx_onedir_post() { + install_macosx_stable_post || return 1 + return 0 +} + install_macosx_git_post() { install_macosx_stable_post || return 1 return 0 @@ -7571,8 +9171,15 @@ install_macosx_git_post() { install_macosx_restart_daemons() { [ $_START_DAEMONS -eq $BS_FALSE ] && return - /bin/launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist || return 1 - /bin/launchctl load -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist || return 1 + if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then + /bin/launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist || return 1 + /bin/launchctl load -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist || return 1 + fi + + if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then + /bin/launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.master.plist || return 1 + /bin/launchctl load -w /Library/LaunchDaemons/com.saltstack.salt.master.plist || return 1 + fi return 0 } @@ -7774,6 +9381,43 @@ preseed_master() { # ####################################################################################################################### +####################################################################################################################### +# +# This function checks if all of the installed daemons are running or not. +# +daemons_running_onedir() { + [ "$_START_DAEMONS" -eq $BS_FALSE ] && return 0 + + FAILED_DAEMONS=0 + for fname in api master minion syndic; do + # Skip salt-api since the service should be opt-in and not necessarily started on boot + [ $fname = "api" ] && continue + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + + if [ -f "/opt/saltstack/salt/run/run" ]; then + salt_path="/opt/saltstack/salt/run/run ${fname}" + else + salt_path="salt-${fname}" + fi + process_running=$(pgrep -f "${salt_path}") + if [ "${process_running}" = "" ]; then + echoerror "${salt_path} was not found running" + FAILED_DAEMONS=$((FAILED_DAEMONS + 1)) + fi + done + + return $FAILED_DAEMONS +} + +# +# Ended daemons running check function +# +####################################################################################################################### + ####################################################################################################################### # # This function checks if all of the installed daemons are running or not. @@ -7874,6 +9518,7 @@ echodebug "PRESEED_MASTER_FUNC=${PRESEED_MASTER_FUNC}" INSTALL_FUNC_NAMES="install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}_${ITYPE}" INSTALL_FUNC_NAMES="$INSTALL_FUNC_NAMES install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}_${ITYPE}" INSTALL_FUNC_NAMES="$INSTALL_FUNC_NAMES install_${DISTRO_NAME_L}_${ITYPE}" +echodebug "INSTALL_FUNC_NAMES=${INSTALL_FUNC_NAMES}" INSTALL_FUNC="null" for FUNC_NAME in $(__strip_duplicates "$INSTALL_FUNC_NAMES"); do @@ -7925,6 +9570,7 @@ DAEMONS_RUNNING_FUNC_NAMES="$DAEMONS_RUNNING_FUNC_NAMES daemons_running_${DISTRO DAEMONS_RUNNING_FUNC_NAMES="$DAEMONS_RUNNING_FUNC_NAMES daemons_running_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}" DAEMONS_RUNNING_FUNC_NAMES="$DAEMONS_RUNNING_FUNC_NAMES daemons_running_${DISTRO_NAME_L}_${ITYPE}" DAEMONS_RUNNING_FUNC_NAMES="$DAEMONS_RUNNING_FUNC_NAMES daemons_running_${DISTRO_NAME_L}" +DAEMONS_RUNNING_FUNC_NAMES="$DAEMONS_RUNNING_FUNC_NAMES daemons_running_${ITYPE}" DAEMONS_RUNNING_FUNC_NAMES="$DAEMONS_RUNNING_FUNC_NAMES daemons_running" DAEMONS_RUNNING_FUNC="null" @@ -8114,6 +9760,11 @@ if [ "$DAEMONS_RUNNING_FUNC" != "null" ] && [ ${_START_DAEMONS} -eq $BS_TRUE ]; fi fi +if [ "$_AUTO_ACCEPT_MINION_KEYS" -eq "$BS_TRUE" ]; then + echoinfo "Accepting the Salt Minion Keys" + salt-key -yA +fi + # Done! if [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then echoinfo "Salt installed!" @@ -8121,6 +9772,13 @@ else echoinfo "Salt configured!" fi +if [ "$_QUICK_START" -eq "$BS_TRUE" ]; then + echoinfo "Congratulations!" + echoinfo "A couple of commands to try:" + echoinfo " salt \* test.ping" + echoinfo " salt \* test.version" +fi + exit 0 # vim: set sts=4 ts=4 et diff --git a/setup/so-functions b/setup/so-functions index 1c9b0f43d..b64daaa92 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1258,7 +1258,7 @@ generate_ssl() { # if the install type is a manager then we need to wait for the minion to be ready before trying # to run the ssl state since we need the minion to sign the certs if [[ "$install_type" =~ ^(EVAL|MANAGER|MANAGERSEARCH|STANDALONE|IMPORT|HELIXSENSOR)$ ]]; then - wait_for_salt_minion + wait_for_salt_minion "$MINION_ID" "5" "$setup_log" || fail_setup fi info "Applying SSL state" logCmd "salt-call state.apply ssl -l info" @@ -1972,6 +1972,7 @@ securityonion_repo() { } repo_sync_local() { + SALTVERSION=$(egrep 'version: [0-9]{4}' ../salt/salt/master.defaults.yaml | sed 's/^.*version: //') info "Repo Sync" if [[ $is_supported ]]; then # Sync the repo from the the SO repo locally. @@ -2021,7 +2022,7 @@ repo_sync_local() { curl -fsSL https://repo.securityonion.net/file/so-repo/prod/2.4/so/so.repo | tee /etc/yum.repos.d/so.repo rpm --import https://repo.saltproject.io/salt/py3/redhat/9/x86_64/SALT-PROJECT-GPG-PUBKEY-2023.pub dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo - curl -fsSL https://repo.saltproject.io/salt/py3/redhat/9/x86_64/minor/3006.1.repo | tee /etc/yum.repos.d/salt.repo + curl -fsSL "https://repo.saltproject.io/salt/py3/redhat/9/x86_64/minor/$SALTVERSION.repo" | tee /etc/yum.repos.d/salt.repo dnf repolist curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install else From c955f9210a16f9ce2105620f9728a53a1d21942d Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 27 Oct 2023 17:24:27 -0400 Subject: [PATCH 29/78] Remove policy for Cases indices --- salt/elasticsearch/defaults.yaml | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 44cb0ea7d..2e19c50b7 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -108,8 +108,6 @@ elasticsearch: match_mapping_type: string settings: index: - lifecycle: - name: so-case-logs mapping: total_fields: limit: 1500 @@ -119,30 +117,6 @@ elasticsearch: sort: field: '@timestamp' order: desc - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 30d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d so-common: close: 30 delete: 365 From 76dd6f07abbe45b08823a615e1a277e5368f5bf8 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 27 Oct 2023 17:26:33 -0400 Subject: [PATCH 30/78] Remove policy for OSQuery manager indices --- salt/elasticsearch/defaults.yaml | 52 -------------------------------- 1 file changed, 52 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 2e19c50b7..2781d2144 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -6323,33 +6323,7 @@ elasticsearch: template: settings: index: - lifecycle: - name: so-logs-osquery-manager-action.responses-logs number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 30d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d so-logs-osquery-manager-actions: index_sorting: false index_template: @@ -6366,33 +6340,7 @@ elasticsearch: template: settings: index: - lifecycle: - name: so-logs-osquery-manager-actions-logs number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 30d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d so-logs-panw_x_panos: index_sorting: false index_template: From 07e51121ba5d2649a75ee12087ece09c7c7e2cb3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 30 Oct 2023 16:11:36 -0400 Subject: [PATCH 31/78] ensure networkminer is latest version --- salt/desktop/packages.sls | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/desktop/packages.sls b/salt/desktop/packages.sls index 3817f2e80..841d34744 100644 --- a/salt/desktop/packages.sls +++ b/salt/desktop/packages.sls @@ -346,7 +346,6 @@ desktop_packages: - snappy - sound-theme-freedesktop - soundtouch - - securityonion-networkminer - speech-dispatcher - speech-dispatcher-espeak-ng - speex @@ -433,6 +432,10 @@ desktop_packages: - xorg-x11-xinit-session - zip +install_networkminer: + pkg.latest: + - name: securityonion-networkminer + {% else %} desktop_packages_os_fail: From ed6473a34b7da12fa6cc1ac13a77dfcc07c39dc9 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 30 Oct 2023 20:41:49 -0400 Subject: [PATCH 32/78] Add roles for eval mode --- salt/elasticsearch/defaults.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 2781d2144..dc8f97e44 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -9088,7 +9088,13 @@ elasticsearch: so-eval: config: node: - roles: [] + roles: + - master + - data + - data_hot + - ingest + - transform + - remote_cluster_client so-heavynode: config: node: From c354924b6853a88050f58bd9babddbbfa7ef2b7d Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 31 Oct 2023 10:05:29 -0400 Subject: [PATCH 33/78] Add import roles --- salt/elasticsearch/defaults.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index dc8f97e44..cd8ff9397 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -9107,7 +9107,13 @@ elasticsearch: so-import: config: node: - roles: [] + roles: + - master + - data + - data_hot + - ingest + - transform + - remote_cluster_client so-manager: config: node: From c420e198fb99dcc9a1faf14bb7d69380d517d180 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 31 Oct 2023 11:18:39 -0400 Subject: [PATCH 34/78] ignore specific Suricata errors --- salt/common/tools/sbin/so-log-check | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 395f60c7d..10f7e8c89 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -158,6 +158,8 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|repodownload.conf" # known issue with reposync on pre-2.4.20 EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing versions record" # stenographer corrupt index EXCLUDED_ERRORS="$EXCLUDED_ERRORS|soc.field." # known ingest type collisions issue with earlier versions of SO + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error parsing signature" # Malformed Suricata rule, from upstream provider + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sticky buffer has no matches" # Non-critical Suricata error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed" From 0c98bd96c7f731c20ed0b5c356fb9f2d2ed32e30 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Oct 2023 12:52:00 -0400 Subject: [PATCH 35/78] Delete salt/idstools/tools/sbin/so-rule UI does this now --- salt/idstools/tools/sbin/so-rule | 454 ------------------------------- 1 file changed, 454 deletions(-) delete mode 100755 salt/idstools/tools/sbin/so-rule diff --git a/salt/idstools/tools/sbin/so-rule b/salt/idstools/tools/sbin/so-rule deleted file mode 100755 index 19618c9f5..000000000 --- a/salt/idstools/tools/sbin/so-rule +++ /dev/null @@ -1,454 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -""" -Local exit codes: - - General error: 1 - - Invalid argument: 2 - - File error: 3 -""" - -import sys, os, subprocess, argparse, signal -import copy -import re -import textwrap -import yaml - -minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions' -salt_proc: subprocess.CompletedProcess = None - - -def print_err(string: str): - print(string, file=sys.stderr) - - -def check_apply(args: dict, prompt: bool = True): - if args.apply: - print('Configuration updated. Applying changes:') - return apply() - else: - if prompt: - message = 'Configuration updated. Would you like to apply your changes now? (y/N) ' - answer = input(message) - while answer.lower() not in [ 'y', 'n', '' ]: - answer = input(message) - if answer.lower() in [ 'n', '' ]: - return 0 - else: - print('Applying changes:') - return apply() - else: - return 0 - - -def apply(): - salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', 'idstools.sync_files', 'queue=True'] - update_cmd = ['so-rule-update'] - print('Syncing config files...') - cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) - if cmd.returncode == 0: - print('Updating rules...') - return subprocess.run(update_cmd).returncode - else: - return cmd.returncode - - -def find_minion_pillar() -> str: - regex = '^.*_(manager|managersearch|standalone|import|eval)\.sls$' - - result = [] - for root, _, files in os.walk(minion_pillar_dir): - for f_minion_id in files: - if re.search(regex, f_minion_id): - result.append(os.path.join(root, f_minion_id)) - - if len(result) == 0: - print_err('Could not find manager-type pillar (eval, standalone, manager, managersearch, import). Are you running this script on the manager?') - sys.exit(3) - elif len(result) > 1: - res_arr = [] - for r in result: - res_arr.append(f'\"{r}\"') - res_str = ', '.join(res_arr) - print_err('(This should not happen, the system is in an error state if you see this message.)\n') - print_err('More than one manager-type pillar exists, minion id\'s listed below:') - print_err(f' {res_str}') - sys.exit(3) - else: - return result[0] - - -def read_pillar(pillar: str): - try: - with open(pillar, 'r') as f: - loaded_yaml = yaml.safe_load(f.read()) - if loaded_yaml is None: - print_err(f'Could not parse {pillar}') - sys.exit(3) - return loaded_yaml - except: - print_err(f'Could not open {pillar}') - sys.exit(3) - - -def write_pillar(pillar: str, content: dict): - try: - sids = content['idstools']['sids'] - if sids['disabled'] is not None: - if len(sids['disabled']) == 0: sids['disabled'] = None - if sids['enabled'] is not None: - if len(sids['enabled']) == 0: sids['enabled'] = None - if sids['modify'] is not None: - if len(sids['modify']) == 0: sids['modify'] = None - - with open(pillar, 'w') as f: - return yaml.dump(content, f, default_flow_style=False) - except Exception as e: - print_err(f'Could not open {pillar}') - sys.exit(3) - - -def check_sid_pattern(sid_pattern: str): - message = f'SID {sid_pattern} is not valid, did you forget the \"re:\" prefix for a regex pattern?' - - if sid_pattern.startswith('re:'): - r_string = sid_pattern[3:] - if not valid_regex(r_string): - print_err('Invalid regex pattern.') - return False - else: - return True - else: - sid: int - try: - sid = int(sid_pattern) - except: - print_err(message) - return False - - if sid >= 0: - return True - else: - print_err(message) - return False - - -def valid_regex(pattern: str): - try: - re.compile(pattern) - return True - except re.error: - return False - - -def sids_key_exists(pillar: dict, key: str): - return key in pillar.get('idstools', {}).get('sids', {}) - - -def rem_from_sids(pillar: dict, key: str, val: str, optional = False): - pillar_dict = copy.deepcopy(pillar) - arr = pillar_dict['idstools']['sids'][key] - if arr is None or val not in arr: - if not optional: print(f'{val} already does not exist in {key}') - else: - pillar_dict['idstools']['sids'][key].remove(val) - return pillar_dict - - -def add_to_sids(pillar: dict, key: str, val: str, optional = False): - pillar_dict = copy.deepcopy(pillar) - if pillar_dict['idstools']['sids'][key] is None: - pillar_dict['idstools']['sids'][key] = [] - if val in pillar_dict['idstools']['sids'][key]: - if not optional: print(f'{val} already exists in {key}') - else: - pillar_dict['idstools']['sids'][key].append(val) - return pillar_dict - - -def add_rem_disabled(args: dict): - global salt_proc - - if not check_sid_pattern(args.sid_pattern): - return 2 - - pillar_dict = read_pillar(args.pillar) - - if not sids_key_exists(pillar_dict, 'disabled'): - pillar_dict['idstools']['sids']['disabled'] = None - - if args.remove: - temp_pillar_dict = rem_from_sids(pillar_dict, 'disabled', args.sid_pattern) - else: - temp_pillar_dict = add_to_sids(pillar_dict, 'disabled', args.sid_pattern) - - if temp_pillar_dict['idstools']['sids']['disabled'] == pillar_dict['idstools']['sids']['disabled']: - salt_proc = check_apply(args, prompt=False) - return salt_proc - else: - pillar_dict = temp_pillar_dict - - if not args.remove: - if sids_key_exists(pillar_dict, 'enabled'): - pillar_dict = rem_from_sids(pillar_dict, 'enabled', args.sid_pattern, optional=True) - - modify = pillar_dict.get('idstools', {}).get('sids', {}).get('modify') - if modify is not None: - rem_candidates = [] - for action in modify: - if action.startswith(f'{args.sid_pattern} '): - rem_candidates.append(action) - if len(rem_candidates) > 0: - for item in rem_candidates: - print(f' - {item}') - answer = input(f'The above modify actions contain {args.sid_pattern}. Would you like to remove them? (Y/n) ') - while answer.lower() not in [ 'y', 'n', '' ]: - for item in rem_candidates: - print(f' - {item}') - answer = input(f'The above modify actions contain {args.sid_pattern}. Would you like to remove them? (Y/n) ') - if answer.lower() in [ 'y', '' ]: - for item in rem_candidates: - modify.remove(item) - pillar_dict['idstools']['sids']['modify'] = modify - - write_pillar(pillar=args.pillar, content=pillar_dict) - - salt_proc = check_apply(args) - return salt_proc - - -def list_disabled_rules(args: dict): - pillar_dict = read_pillar(args.pillar) - - disabled = pillar_dict.get('idstools', {}).get('sids', {}).get('disabled') - if disabled is None: - print('No rules disabled.') - return 0 - else: - print('Disabled rules:') - for rule in disabled: - print(f' - {rule}') - return 0 - - -def add_rem_enabled(args: dict): - global salt_proc - - if not check_sid_pattern(args.sid_pattern): - return 2 - - pillar_dict = read_pillar(args.pillar) - - if not sids_key_exists(pillar_dict, 'enabled'): - pillar_dict['idstools']['sids']['enabled'] = None - - if args.remove: - temp_pillar_dict = rem_from_sids(pillar_dict, 'enabled', args.sid_pattern) - else: - temp_pillar_dict = add_to_sids(pillar_dict, 'enabled', args.sid_pattern) - - if temp_pillar_dict['idstools']['sids']['enabled'] == pillar_dict['idstools']['sids']['enabled']: - salt_proc = check_apply(args, prompt=False) - return salt_proc - else: - pillar_dict = temp_pillar_dict - - if not args.remove: - if sids_key_exists(pillar_dict, 'disabled'): - pillar_dict = rem_from_sids(pillar_dict, 'disabled', args.sid_pattern, optional=True) - - write_pillar(pillar=args.pillar, content=pillar_dict) - - salt_proc = check_apply(args) - return salt_proc - - -def list_enabled_rules(args: dict): - pillar_dict = read_pillar(args.pillar) - - enabled = pillar_dict.get('idstools', {}).get('sids', {}).get('enabled') - if enabled is None: - print('No rules explicitly enabled.') - return 0 - else: - print('Enabled rules:') - for rule in enabled: - print(f' - {rule}') - return 0 - - -def add_rem_modify(args: dict): - global salt_proc - - if not check_sid_pattern(args.sid_pattern): - return 2 - - if not valid_regex(args.search_term): - print_err('Search term is not a valid regex pattern.') - - string_val = f'{args.sid_pattern} \"{args.search_term}\" \"{args.replace_term}\"' - - pillar_dict = read_pillar(args.pillar) - - if not sids_key_exists(pillar_dict, 'modify'): - pillar_dict['idstools']['sids']['modify'] = None - - if args.remove: - temp_pillar_dict = rem_from_sids(pillar_dict, 'modify', string_val) - else: - temp_pillar_dict = add_to_sids(pillar_dict, 'modify', string_val) - - if temp_pillar_dict['idstools']['sids']['modify'] == pillar_dict['idstools']['sids']['modify']: - salt_proc = check_apply(args, prompt=False) - return salt_proc - else: - pillar_dict = temp_pillar_dict - - # TODO: Determine if a rule should be removed from disabled if modified. - if not args.remove: - if sids_key_exists(pillar_dict, 'disabled'): - pillar_dict = rem_from_sids(pillar_dict, 'disabled', args.sid_pattern, optional=True) - - write_pillar(pillar=args.pillar, content=pillar_dict) - - salt_proc = check_apply(args) - return salt_proc - - -def list_modified_rules(args: dict): - pillar_dict = read_pillar(args.pillar) - - modify = pillar_dict.get('idstools', {}).get('sids', {}).get('modify') - if modify is None: - print('No rules currently modified.') - return 0 - else: - print('Modified rules + modifications:') - for rule in modify: - print(f' - {rule}') - return 0 - - -def sigint_handler(*_): - print('Exiting gracefully on Ctrl-C') - if salt_proc is not None: salt_proc.send_signal(signal.SIGINT) - sys.exit(0) - - -def main(): - signal.signal(signal.SIGINT, sigint_handler) - - if os.geteuid() != 0: - print_err('You must run this script as root') - sys.exit(1) - - apply_help='After updating rule configuration, apply the idstools state.' - - main_parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter) - - subcommand_desc = textwrap.dedent( - """\ - disabled Manage and list disabled rules (add, remove, list) - enabled Manage and list enabled rules (add, remove, list) - modify Manage and list modified rules (add, remove, list) - """ - ) - subparsers = main_parser.add_subparsers(title='commands', description=subcommand_desc, metavar='', dest='command') - - - sid_or_regex_help = 'A valid SID (ex: "4321") or regular expression pattern (ex: "re:heartbleed|spectre")' - - # Disabled actions - disabled = subparsers.add_parser('disabled') - disabled_sub = disabled.add_subparsers() - - disabled_add = disabled_sub.add_parser('add') - disabled_add.set_defaults(func=add_rem_disabled) - disabled_add.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help) - disabled_add.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help) - - disabled_rem = disabled_sub.add_parser('remove') - disabled_rem.set_defaults(func=add_rem_disabled, remove=True) - disabled_rem.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help) - disabled_rem.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help) - - disabled_list = disabled_sub.add_parser('list') - disabled_list.set_defaults(func=list_disabled_rules) - - - # Enabled actions - enabled = subparsers.add_parser('enabled') - enabled_sub = enabled.add_subparsers() - - enabled_add = enabled_sub.add_parser('add') - enabled_add.set_defaults(func=add_rem_enabled) - enabled_add.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help) - enabled_add.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help) - - enabled_rem = enabled_sub.add_parser('remove') - enabled_rem.set_defaults(func=add_rem_enabled, remove=True) - enabled_rem.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help) - enabled_rem.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help) - - enabled_list = enabled_sub.add_parser('list') - enabled_list.set_defaults(func=list_enabled_rules) - - - search_term_help='A properly escaped regex search term (ex: "\\\$EXTERNAL_NET")' - replace_term_help='The text to replace the search term with' - - # Modify actions - modify = subparsers.add_parser('modify') - modify_sub = modify.add_subparsers() - - modify_add = modify_sub.add_parser('add') - modify_add.set_defaults(func=add_rem_modify) - modify_add.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help) - modify_add.add_argument('search_term', metavar='SEARCH_TERM', help=search_term_help) - modify_add.add_argument('replace_term', metavar='REPLACE_TERM', help=replace_term_help) - modify_add.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help) - - modify_rem = modify_sub.add_parser('remove') - modify_rem.set_defaults(func=add_rem_modify, remove=True) - modify_rem.add_argument('sid_pattern', metavar='SID', help=sid_or_regex_help) - modify_rem.add_argument('search_term', metavar='SEARCH_TERM', help=search_term_help) - modify_rem.add_argument('replace_term', metavar='REPLACE_TERM', help=replace_term_help) - modify_rem.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help) - - modify_list = modify_sub.add_parser('list') - modify_list.set_defaults(func=list_modified_rules) - - - # Begin parse + run - args = main_parser.parse_args(sys.argv[1:]) - - if not hasattr(args, 'remove'): - args.remove = False - - args.pillar = find_minion_pillar() - - if hasattr(args, 'func'): - exit_code = args.func(args) - else: - if args.command is None: - main_parser.print_help() - else: - if args.command == 'disabled': - disabled.print_help() - elif args.command == 'enabled': - enabled.print_help() - elif args.command == 'modify': - modify.print_help() - sys.exit(0) - - sys.exit(exit_code) - - -if __name__ == '__main__': - main() From cc3a69683c58d0e0618713a081dd5bcf66ae8fb7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Oct 2023 12:55:47 -0400 Subject: [PATCH 36/78] Delete salt/manager/tools/sbin/so-allow-view --- salt/manager/tools/sbin/so-allow-view | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100755 salt/manager/tools/sbin/so-allow-view diff --git a/salt/manager/tools/sbin/so-allow-view b/salt/manager/tools/sbin/so-allow-view deleted file mode 100755 index 58b972ee2..000000000 --- a/salt/manager/tools/sbin/so-allow-view +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -echo "" -echo "Hosts/Networks that have access to login to the Security Onion Console:" - -so-firewall includedhosts analyst From 497294c363d721331e9f727d6d1eacbb20bdb202 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Oct 2023 12:57:10 -0400 Subject: [PATCH 37/78] Delete salt/common/tools/sbin/so-zeek-logs --- salt/common/tools/sbin/so-zeek-logs | 67 ----------------------------- 1 file changed, 67 deletions(-) delete mode 100755 salt/common/tools/sbin/so-zeek-logs diff --git a/salt/common/tools/sbin/so-zeek-logs b/salt/common/tools/sbin/so-zeek-logs deleted file mode 100755 index f6df7f8aa..000000000 --- a/salt/common/tools/sbin/so-zeek-logs +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -local_salt_dir=/opt/so/saltstack/local - -zeek_logs_enabled() { - echo "zeeklogs:" > $local_salt_dir/pillar/zeeklogs.sls - echo " enabled:" >> $local_salt_dir/pillar/zeeklogs.sls - for BLOG in "${BLOGS[@]}"; do - echo " - $BLOG" | tr -d '"' >> $local_salt_dir/pillar/zeeklogs.sls - done -} - -whiptail_manager_adv_service_zeeklogs() { - BLOGS=$(whiptail --title "so-zeek-logs" --checklist "Please Select Logs to Send:" 24 78 12 \ - "conn" "Connection Logging" ON \ - "dce_rpc" "RPC Logs" ON \ - "dhcp" "DHCP Logs" ON \ - "dnp3" "DNP3 Logs" ON \ - "dns" "DNS Logs" ON \ - "dpd" "DPD Logs" ON \ - "files" "Files Logs" ON \ - "ftp" "FTP Logs" ON \ - "http" "HTTP Logs" ON \ - "intel" "Intel Hits Logs" ON \ - "irc" "IRC Chat Logs" ON \ - "kerberos" "Kerberos Logs" ON \ - "modbus" "MODBUS Logs" ON \ - "notice" "Zeek Notice Logs" ON \ - "ntlm" "NTLM Logs" ON \ - "pe" "PE Logs" ON \ - "radius" "Radius Logs" ON \ - "rfb" "RFB Logs" ON \ - "rdp" "RDP Logs" ON \ - "sip" "SIP Logs" ON \ - "smb_files" "SMB Files Logs" ON \ - "smb_mapping" "SMB Mapping Logs" ON \ - "smtp" "SMTP Logs" ON \ - "snmp" "SNMP Logs" ON \ - "ssh" "SSH Logs" ON \ - "ssl" "SSL Logs" ON \ - "syslog" "Syslog Logs" ON \ - "tunnel" "Tunnel Logs" ON \ - "weird" "Zeek Weird Logs" ON \ - "mysql" "MySQL Logs" ON \ - "socks" "SOCKS Logs" ON \ - "x509" "x.509 Logs" ON 3>&1 1>&2 2>&3 ) - - local exitstatus=$? - - IFS=' ' read -ra BLOGS <<< "$BLOGS" - - return $exitstatus -} - -whiptail_manager_adv_service_zeeklogs -return_code=$? -case $return_code in - 1) - whiptail --title "so-zeek-logs" --msgbox "Cancelling. No changes have been made." 8 75 - ;; - 255) - whiptail --title "so-zeek-logs" --msgbox "Whiptail error occured, exiting." 8 75 - ;; - *) - zeek_logs_enabled - ;; -esac - From d07cfdd3fe9366a6c22f6725027a05ec1be7af82 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Oct 2023 13:10:55 -0400 Subject: [PATCH 38/78] Update so-functions --- setup/so-functions | 182 --------------------------------------------- 1 file changed, 182 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 42402ad86..56c4b29c1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -268,15 +268,6 @@ collect_dockernet() { fi } -collect_es_space_limit() { - whiptail_log_size_limit "$log_size_limit" - - while ! valid_int "$log_size_limit"; do # Upper/lower bounds? - whiptail_invalid_input - whiptail_log_size_limit "$log_size_limit" - done -} - collect_gateway() { whiptail_management_interface_gateway @@ -286,28 +277,6 @@ collect_gateway() { done } -collect_homenet_mngr() { - whiptail_homenet_manager "10.0.0.0/8,192.168.0.0/16,172.16.0.0/12" - - while ! valid_cidr_list "$HNMANAGER"; do - whiptail_invalid_input - whiptail_homenet_manager "$HNMANAGER" - done -} - -collect_homenet_snsr() { - if whiptail_homenet_sensor_inherit; then - export HNSENSOR=inherit - else - whiptail_homenet_sensor "10.0.0.0/8,192.168.0.0/16,172.16.0.0/12" - - while ! valid_cidr_list "$HNSENSOR"; do - whiptail_invalid_input - whiptail_homenet_sensor "$HNSENSOR" - done - fi -} - collect_hostname() { collect_hostname_validate @@ -346,26 +315,6 @@ collect_idh_preferences() { if [[ "$idh_preferences" != "" ]]; then IDH_MGTRESTRICT='True'; fi } -collect_idh_services() { - whiptail_idh_services - - case "$IDH_SERVICES" in - 'Linux Webserver (NAS Skin)') - IDH_SERVICES='"HTTP","FTP","SSH"' - ;; - 'MySQL Server') - IDH_SERVICES='"MYSQL","SSH"' - ;; - 'MSSQL Server') - IDH_SERVICES='"MSSQL","VNC' - ;; - 'Custom') - whiptail_idh_services_custom - IDH_SERVICES=$(echo "$IDH_SERVICES" | tr '[:blank:]' ',' ) - ;; - esac -} - collect_int_ip_mask() { whiptail_management_interface_ip_mask @@ -425,71 +374,6 @@ collect_net_method() { fi } -collect_ntp_servers() { - if whiptail_ntp_ask; then - [[ $is_airgap ]] && ntp_string="" - whiptail_ntp_servers "$ntp_string" - - while ! valid_ntp_list "$ntp_string"; do - whiptail_invalid_input - whiptail_ntp_servers "$ntp_string" - done - - IFS="," read -r -a ntp_servers <<< "$ntp_string" # Split string on commas into array - else - ntp_servers=() - fi -} - -collect_oinkcode() { - whiptail_oinkcode - - while ! valid_string "$OINKCODE" "" "128"; do - whiptail_invalid_input - whiptail_oinkcode "$OINKCODE" - done -} - -collect_patch_schedule() { - whiptail_patch_schedule - - case "$patch_schedule" in - 'New Schedule') - whiptail_patch_schedule_select_days - whiptail_patch_schedule_select_hours - collect_patch_schedule_name_new - patch_schedule_os_new - ;; - 'Import Schedule') - collect_patch_schedule_name_import - ;; - 'Automatic') - PATCHSCHEDULENAME='auto' - ;; - 'Manual') - PATCHSCHEDULENAME='manual' - ;; - esac -} - -collect_patch_schedule_name_new() { - whiptail_patch_name_new_schedule - - while ! valid_string "$PATCHSCHEDULENAME"; do - whiptail_invalid_string "schedule name" - whiptail_patch_name_new_schedule "$PATCHSCHEDULENAME" - done -} - -collect_patch_schedule_name_import() { - whiptail_patch_schedule_import - - while ! valid_string "$PATCHSCHEDULENAME"; do - whiptail_invalid_string "schedule name" - whiptail_patch_schedule_import "$PATCHSCHEDULENAME" - done -} - collect_proxy() { [[ -n $TESTING ]] && return local ask=${1:-true} @@ -658,47 +542,6 @@ configure_minion() { } >> "$setup_log" 2>&1 } -configure_ntp() { - local chrony_conf=/etc/chrony.conf - - # Install chrony if it isn't already installed - if ! command -v chronyc &> /dev/null; then - logCmd "dnf -y install chrony" - fi - - [[ -f $chrony_conf ]] && mv $chrony_conf "$chrony_conf.bak" - - printf '%s\n' "# NTP server list" > $chrony_conf - - # Build list of servers - for addr in "${ntp_servers[@]}"; do - echo "server $addr iburst" >> $chrony_conf - done - - printf '\n%s\n' "# Config options" >> $chrony_conf - - printf '%s\n' \ - 'driftfile /var/lib/chrony/drift' \ - 'makestep 1.0 3' \ - 'rtcsync' \ - 'logdir /var/log/chrony' >> $chrony_conf - - if [[ $is_rpm ]]; then - systemctl enable chronyd - systemctl restart chronyd - else - systemctl enable chrony - systemctl restart chrony - fi - - # Tell the chrony daemon to sync time & update the system time - # Since these commands only make a call to chronyd, wait after each command to make sure the changes are made - printf "Syncing chrony time to server: " - chronyc -a 'burst 4/4' && sleep 30 - printf "Forcing chrony to update the time: " - chronyc -a makestep && sleep 30 -} - checkin_at_boot() { local minion_config=/etc/salt/minion @@ -1055,16 +898,6 @@ download_elastic_agent_artifacts() { fi } -installer_progress_loop() { - local i=0 - local msg="${1:-Performing background actions...}" - while true; do - [[ $i -lt 98 ]] && ((i++)) - set_progress_str "$i" "$msg" nolog - [[ $i -gt 0 ]] && sleep 5s - done -} - installer_prereq_packages() { if [[ $is_deb ]]; then # Print message to stdout so the user knows setup is doing something @@ -1648,21 +1481,6 @@ network_setup() { logCmd "sed -i '/\$MNIC/${INTERFACE}/g' /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable" } -ntp_pillar_entries() { - - local pillar_file=$local_salt_dir/pillar/minions/$MINION_ID.sls - - - if [[ ${#ntp_servers[@]} -gt 0 ]]; then - printf '%s\n'\ - "ntp:"\ - " servers:" > "$pillar_file" - for addr in "${ntp_servers[@]}"; do - printf '%s\n' " - '$addr'" >> "$pillar_file" - done - fi -} - parse_install_username() { # parse out the install username so things copy correctly INSTALLUSERNAME=${SUDO_USER:-${USER}} From ae45d40eca779ebb06403b3ce5ffabc1572f32e9 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 13:34:30 +0000 Subject: [PATCH 39/78] Add Sublime Platform ingest pipeline --- salt/elasticsearch/files/ingest/sublime | 34 +++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 salt/elasticsearch/files/ingest/sublime diff --git a/salt/elasticsearch/files/ingest/sublime b/salt/elasticsearch/files/ingest/sublime new file mode 100644 index 000000000..c26f93c01 --- /dev/null +++ b/salt/elasticsearch/files/ingest/sublime @@ -0,0 +1,34 @@ +{ + "description" : " Email alerts from Sublime", + "processors" : [ + { "set": { "field": "event.module", "value": "sublime" } }, + { "set": { "field": "event.dataset", "value": "alert" } }, + { "set": { "field": "event.severity", "value": 3, "override": true } }, + { "set": { "field": "rule.name", "value": "Sublime Platform: {{ flagged_rules.0.name }}", "override": true } }, + { "set": { "field": "sublime.message_group_id", "value": "{{ _id }}", "override": true } }, + { "set": { "field": "email.address", "value": "{{ messages.0.recipients.0.email }}", "override": true } }, + { "set": { "field": "email.forwarded_recipents", "value": "{{ messages.0.forwarded_receipients }}", "override": true } }, + { "set": { "field": "email.sender.address", "value": "{{ messages.0.sender.email }}", "override": true } }, + { "set": { "field": "email.subject", "value": "{{ messages.0.subject }}", "override": true } }, + { "set": { "field": "email.forwarded_at", "value": "{{ messages.0.forwarded_at }}", "override": true } }, + { "set": { "field": "email.created_at", "value": "{{ messages.0.created_at }}", "override": true } }, + { "set": { "field": "email.read_at", "value": "{{ messages.0.read_at }}", "override": true } }, + { "set": { "field": "email.replied_at", "value": "{{ messages.0.replied_at }}", "override": true } }, + { + "grok": { + "field": "sublime.request_url", + "patterns": ["^https://api.%{DATA:sublime_host}/v0%{GREEDYDATA}$"], + "ignore_failure": true + } + }, + + { "rename": { "field": "sublime_host", "target_field": "sublime.url", "ignore_missing": true } }, + { "rename": { "field": "data", "target_field": "sublime", "ignore_missing": true } }, + { "rename": { "field": "flagged_rules", "target_field": "sublime.flagged_rules", "ignore_missing": true } }, + { "rename": { "field": "organization_id", "target_field": "sublime.organization_id", "ignore_missing": true } }, + { "rename": { "field": "review_status", "target_field": "sublime.review_status", "ignore_missing": true } }, + { "rename": { "field": "state", "target_field": "sublime.state", "ignore_missing": true } }, + { "rename": { "field": "user_reports", "target_field": "sublime.user_reports", "ignore_missing": true } }, + { "pipeline": { "name": "common" } } + ] +} From 4dc64400c58b9de09f6ac811180a55208dcea7a6 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 13:36:32 +0000 Subject: [PATCH 40/78] Support document_id --- .../so/9805_output_elastic_agent.conf.jinja | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja index 0a148155c..f7671e2b7 100644 --- a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja +++ b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja @@ -1,13 +1,16 @@ output { if "elastic-agent" in [tags] { - if [metadata][pipeline] { + if [metadata][pipeline] { + if [metadata][_id] { elasticsearch { hosts => "{{ GLOBALS.manager }}" ecs_compatibility => v8 data_stream => true user => "{{ ES_USER }}" password => "{{ ES_PASS }}" + document_id => "%{[metadata][_id]}" pipeline => "%{[metadata][pipeline]}" + silence_errors_in_log => ["version_conflict_engine_exception"] ssl => true ssl_certificate_verification => false } @@ -19,10 +22,22 @@ output { data_stream => true user => "{{ ES_USER }}" password => "{{ ES_PASS }}" + pipeline => "%{[metadata][pipeline]}" ssl => true - ssl_certificate_verification => false + ssl_certificate_verification => false } - } + } + } + else { + elasticsearch { + hosts => "{{ GLOBALS.manager }}" + ecs_compatibility => v8 + data_stream => true + user => "{{ ES_USER }}" + password => "{{ ES_PASS }}" + ssl => true + ssl_certificate_verification => false + } + } } } - From 51247be6b97583e15dad1af1c93ff031ef0a51a9 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 13:37:52 +0000 Subject: [PATCH 41/78] Sublime Platform integration defaults --- salt/elasticfleet/defaults.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index a17957e7c..ba779f5a0 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -90,3 +90,10 @@ elasticfleet: - zscaler_zia - zscaler_zpa - 1password + optional_integrations: + sublime_platform: + enabled_nodes: [] + api_key: + base_url: https://api.platform.sublimesecurity.com + poll_interval: 5m + limit: 100 From 23ee9c2bb02842b7cd745e7b667bc02ad4ea6328 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 13:41:40 +0000 Subject: [PATCH 42/78] Sublime Platform integration --- .../sublime_platform.json | 44 +++++++++++++++++++ salt/elasticfleet/soc_elasticfleet.yaml | 32 ++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 salt/elasticfleet/files/integrations-optional/sublime_platform.json diff --git a/salt/elasticfleet/files/integrations-optional/sublime_platform.json b/salt/elasticfleet/files/integrations-optional/sublime_platform.json new file mode 100644 index 000000000..8feedc879 --- /dev/null +++ b/salt/elasticfleet/files/integrations-optional/sublime_platform.json @@ -0,0 +1,44 @@ +{%- from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED -%} +{%- from 'sensoroni/map.jinja' import SENSORONIMERGED -%} +{%- from 'vars/globals.map.jinja' import GLOBALS -%} +{%- raw -%} +{ + "package": { + "name": "httpjson", + "version": "" + }, + "name": "sublime-platform", + "namespace": "default", + "description": "", + "policy_id": "FleetServer_{%- endraw -%}{{ NAME }}{%- raw -%}", + "vars": {}, + "inputs": { + "generic-httpjson": { + "enabled": true, + "streams": { + "httpjson.generic": { + "enabled": true, + "vars": { + "request_method": "GET", + "processors": "- drop_event:\n when:\n not:\n contains: \n message: \"flagged_rules\"\n- decode_json_fields:\n fields: [\"message\"]\n document_id: id\n target: \"\"", + "enable_request_tracer": false, + "oauth_scopes": [], + "request_transforms": "- set:\n target: header.Authorization\n value: 'Bearer {% endraw -%}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.api_key }}{%- raw -%}'\n- set:\n target: header.accept\n value: application/json\n- set:\n target: url.params.last_message_created_at[gte]\n value: '[[formatDate (now (parseDuration \"-{%- endraw -%}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.poll_interval }}{%- raw -%}\")) \"2006-01-02T15:04:05Z\"]]'\n- set:\n target: url.params.reviewed\n value: false\n- set:\n target: url.params.flagged\n value: true\n- set:\n target: url.params.limit\n value: {% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.limit }}{%- raw -%}", + "response_transforms": "", + "request_redirect_headers_ban_list": [], + "request_encode_as": "application/x-www-form-urlencoded", + "request_url": "{%- endraw -%}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.base_url }}{%- raw -%}/v0/message-groups", + "response_split": "target: body.message_groups\ntype: array\nkeep_parent: false\ntransforms:\n - set:\n target: body.sublime.request_url\n value : '[[ .last_response.url.value ]]'", + "tags": [ + "forwarded" + ], + "pipeline": "sublime", + "data_stream.dataset": "sublime", + "request_interval": "1m" + } + } + } + } + } +} +{%- endraw -%} diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index af660358a..8cb975086 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -40,3 +40,35 @@ elasticfleet: helpLink: elastic-fleet.html sensitive: True advanced: True + optional_integrations: + sublime_platform: + enabled_nodes: + description: Determines if the Sublime Platform integration is enabled. + global: True + helpLink: elastic-fleet.html + advanced: True + forcedType: "[]string" + api_key: + description: API key for Sublime Platform. + global: False + helpLink: elastic-fleet.html + advanced: True + forcedType: string + base_url: + description: Base URL for Sublime Platform. + global: False + helpLink: elastic-fleet.html + advanced: True + forcedType: string + poll_interval: + description: Poll interval for alerts from Sublime Platform. + global: False + helpLink: elastic-fleet.html + advanced: True + forcedType: string + limit: + description: The maximum number of message groups to return from Sublime Platform. + global: False + helpLink: elastic-fleet.html + advanced: True + forcedType: int From 9701d0ac206382124b5e8ffe854267b30f48aa18 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 13:47:20 +0000 Subject: [PATCH 43/78] Optional integration Fleet configuration --- salt/elasticfleet/config.sls | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index d2e357c91..29eb6e972 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -6,6 +6,7 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% if sls.split('.')[0] in allowed_states %} +{% set node_data = salt['pillar.get']('node_data') %} # Add EA Group elasticfleetgroup: @@ -92,6 +93,34 @@ eaintegration: - user: 947 - group: 939 +{% for minion in node_data %} +{% set role = node_data[minion]["role"] %} +{% if role in [ "fleet","heavynode", "manager","managersearch","standalone" ] %} +{% set optional_integrations = salt['pillar.get']('elasticfleet:optional_integrations', {}) %} +{% set integration_keys = salt['pillar.get']('elasticfleet:optional_integrations', {}).keys() %} +fleet_server_integrations_{{ minion }}: + file.directory: + - name: /opt/so/conf/elastic-fleet/integrations/FleetServer_{{ minion }} + - user: 947 + - group: 939 + - makedirs: True +{% for integration in integration_keys %} +{% set enabled_nodes = optional_integrations[integration]["enabled_nodes"] %} +{% if minion in enabled_nodes %} +optional_integrations_dynamic_{{ minion }}: + file.managed: + - name: /opt/so/conf/elastic-fleet/integrations/FleetServer_{{ minion }}/{{ integration }}.json + - source: salt://elasticfleet/files/integrations-optional/{{ integration }}.json + - user: 947 + - group: 939 + - template: jinja + - defaults: + NAME: {{ minion }} +{% endif %} +{% endfor %} +{% endif %} +{% endfor %} + ea-integrations-load: file.absent: - name: /opt/so/state/eaintegrations.txt @@ -99,6 +128,9 @@ ea-integrations-load: - file: eaintegration - file: eadynamicintegration - file: eapackageupgrade + {% for minion in node_data %} + - file: optional_integrations_dynamic_{{ minion }} + {% endfor %} {% endif %} {% else %} From 44e45843bfc5c9af30f54ea01306e2ee5a2408d2 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 13:52:38 +0000 Subject: [PATCH 44/78] Change optional integration Fleet configuration --- salt/elasticfleet/config.sls | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index 29eb6e972..78fb1d412 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -93,6 +93,13 @@ eaintegration: - user: 947 - group: 939 +eaoptionalintegrationsdir: + file.directory: + - name: /opt/so/conf/elastic-fleet/integrations-optional + - user: 947 + - group: 939 + - makedirs: True + {% for minion in node_data %} {% set role = node_data[minion]["role"] %} {% if role in [ "fleet","heavynode", "manager","managersearch","standalone" ] %} @@ -100,16 +107,16 @@ eaintegration: {% set integration_keys = salt['pillar.get']('elasticfleet:optional_integrations', {}).keys() %} fleet_server_integrations_{{ minion }}: file.directory: - - name: /opt/so/conf/elastic-fleet/integrations/FleetServer_{{ minion }} + - name: /opt/so/conf/elastic-fleet/integrations-optional/FleetServer_{{ minion }} - user: 947 - group: 939 - makedirs: True {% for integration in integration_keys %} {% set enabled_nodes = optional_integrations[integration]["enabled_nodes"] %} {% if minion in enabled_nodes %} -optional_integrations_dynamic_{{ minion }}: +optional_integrations_dynamic_{{ minion }}_{{ integration }}: file.managed: - - name: /opt/so/conf/elastic-fleet/integrations/FleetServer_{{ minion }}/{{ integration }}.json + - name: /opt/so/conf/elastic-fleet/integrations-optional/FleetServer_{{ minion }}/{{ integration }}.json - source: salt://elasticfleet/files/integrations-optional/{{ integration }}.json - user: 947 - group: 939 @@ -120,17 +127,13 @@ optional_integrations_dynamic_{{ minion }}: {% endfor %} {% endif %} {% endfor %} - ea-integrations-load: file.absent: - name: /opt/so/state/eaintegrations.txt - onchanges: - file: eaintegration - file: eadynamicintegration - - file: eapackageupgrade - {% for minion in node_data %} - - file: optional_integrations_dynamic_{{ minion }} - {% endfor %} + - file: /opt/so/conf/elastic-fleet/integrations-optional/* {% endif %} {% else %} From a0926b7b872ea091475e71b6476c3732f99256cd Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 13:59:24 +0000 Subject: [PATCH 45/78] Load optional integrations --- .../so-elastic-fleet-integration-policy-load | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index 44e7ccf2b..518d29d26 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -64,8 +64,28 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then if [[ "$RETURN_CODE" != "1" ]]; then touch /opt/so/state/eaintegrations.txt fi + + # Fleet Server - Optional integrations + for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json + do + if ! [ "$INTEGRATION" == "/opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json" ]; then + FLEET_POLICY=`echo "$INTEGRATION"| cut -d'/' -f7` + printf "\n\nFleet Server Policy - Loading $INTEGRATION\n" + elastic_fleet_integration_check "$FLEET_POLICY" "$INTEGRATION" + if [ -n "$INTEGRATION_ID" ]; then + printf "\n\nIntegration $NAME exists - Updating integration\n" + elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION" + else + printf "\n\nIntegration does not exist - Creating integration\n" + if [ "$NAME" != "elasticsearch-logs" ]; then + elastic_fleet_integration_create "@$INTEGRATION" + fi + fi + fi + done + if [[ "$RETURN_CODE" != "1" ]]; then + touch /opt/so/state/eaintegrations.txt + fi else exit $RETURN_CODE fi - - From bca1194a468362f36749cbaeafcbb2f15c98b5b4 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 14:01:55 +0000 Subject: [PATCH 46/78] Sublime SOC Action --- salt/soc/defaults.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6d8ed5bfd..ceca9ef31 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -59,6 +59,12 @@ soc: target: _blank links: - 'https://www.virustotal.com/gui/search/{value}' + - name: Sublime Platform Email Review + description: Review email in Sublime Platform + icon: fa-external-link-alt + target: _blank + links: + - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' eventFields: default: - soc_timestamp From 338146feddf21e1cb0e7faea232d37d318bab2d7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 1 Nov 2023 10:19:56 -0400 Subject: [PATCH 47/78] fix repo update during soup for airgap --- salt/manager/tools/sbin/soup | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f30c3f15d..263fab7d0 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -579,7 +579,7 @@ update_airgap_rules() { rsync -av $UPDATE_DIR/agrules/* /nsm/repo/rules/ } -update_centos_repo() { +update_airgap_repo() { # Update the files in the repo echo "Syncing new updates to /nsm/repo" rsync -av $AGREPO/* /nsm/repo/ @@ -821,6 +821,7 @@ main() { set -e if [[ $is_airgap -eq 0 ]]; then + update_airgap_repo yum clean all check_os_updates elif [[ $OS == 'oracle' ]]; then @@ -895,11 +896,6 @@ main() { update_airgap_rules fi - # Only update the repo if its airgap - if [[ $is_airgap -eq 0 && $UPGRADESALT -ne 1 ]]; then - update_centos_repo - fi - # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars if [[ ! "$MINIONID" =~ "_import" ]]; then echo "" From 2b3e405b2d30361874f2a76f3ff4a4d8cc0b5b31 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Nov 2023 10:41:40 -0400 Subject: [PATCH 48/78] Delete pillar/thresholding/pillar.usage --- pillar/thresholding/pillar.usage | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 pillar/thresholding/pillar.usage diff --git a/pillar/thresholding/pillar.usage b/pillar/thresholding/pillar.usage deleted file mode 100644 index 1626433b1..000000000 --- a/pillar/thresholding/pillar.usage +++ /dev/null @@ -1,20 +0,0 @@ -thresholding: - sids: - : - - threshold: - gen_id: - type: - track: - count: - seconds: - - rate_filter: - gen_id: - track: - count: - seconds: - new_action: - timeout: - - suppress: - gen_id: - track: - ip: From f62e02a47704a3a62b7deca1fc93321349067bd4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Nov 2023 10:42:29 -0400 Subject: [PATCH 49/78] Delete pillar/thresholding/pillar.example --- pillar/thresholding/pillar.example | 44 ------------------------------ 1 file changed, 44 deletions(-) delete mode 100644 pillar/thresholding/pillar.example diff --git a/pillar/thresholding/pillar.example b/pillar/thresholding/pillar.example deleted file mode 100644 index 705cb606c..000000000 --- a/pillar/thresholding/pillar.example +++ /dev/null @@ -1,44 +0,0 @@ -thresholding: - sids: - 8675309: - - threshold: - gen_id: 1 - type: threshold - track: by_src - count: 10 - seconds: 10 - - threshold: - gen_id: 1 - type: limit - track: by_dst - count: 100 - seconds: 30 - - rate_filter: - gen_id: 1 - track: by_rule - count: 50 - seconds: 30 - new_action: alert - timeout: 30 - - suppress: - gen_id: 1 - track: by_either - ip: 10.10.3.7 - 11223344: - - threshold: - gen_id: 1 - type: limit - track: by_dst - count: 10 - seconds: 10 - - rate_filter: - gen_id: 1 - track: by_src - count: 50 - seconds: 20 - new_action: pass - timeout: 60 - - suppress: - gen_id: 1 - track: by_src - ip: 10.10.3.0/24 From 655c88cd09874abe92106b3df38a8ae0225b4a55 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 16:47:51 +0000 Subject: [PATCH 50/78] Make sure enabled_nodes is populated --- salt/elasticfleet/config.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index 78fb1d412..434b6db2d 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -112,6 +112,7 @@ fleet_server_integrations_{{ minion }}: - group: 939 - makedirs: True {% for integration in integration_keys %} +{% if 'enabled_nodes' in optional_integrations[integration]%} {% set enabled_nodes = optional_integrations[integration]["enabled_nodes"] %} {% if minion in enabled_nodes %} optional_integrations_dynamic_{{ minion }}_{{ integration }}: @@ -124,6 +125,7 @@ optional_integrations_dynamic_{{ minion }}_{{ integration }}: - defaults: NAME: {{ minion }} {% endif %} +{% endif %} {% endfor %} {% endif %} {% endfor %} From c32935e2e63fbf6e01bd7417e8afdf9453163993 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 17:02:43 +0000 Subject: [PATCH 51/78] Remove optional integration from configuration if not enabled --- salt/elasticfleet/config.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index 434b6db2d..02672d58f 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -124,6 +124,10 @@ optional_integrations_dynamic_{{ minion }}_{{ integration }}: - template: jinja - defaults: NAME: {{ minion }} +{% else %} +optional_integrations_dynamic_{{ minion }}_{{ integration }}_delete: + file.absent: + - name: /opt/so/conf/elastic-fleet/integrations-optional/FleetServer_{{ minion }}/{{ integration }}.json {% endif %} {% endif %} {% endfor %} From b3b67acf07a9d7099db56a3308b5c70942d4f9c7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Nov 2023 15:11:54 -0400 Subject: [PATCH 52/78] Add memory restrictions --- .../assigned_hostgroups.local.map.yaml | 1 - setup/so-functions | 27 ++++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/files/firewall/assigned_hostgroups.local.map.yaml b/files/firewall/assigned_hostgroups.local.map.yaml index 07f389af0..be34cb0be 100644 --- a/files/firewall/assigned_hostgroups.local.map.yaml +++ b/files/firewall/assigned_hostgroups.local.map.yaml @@ -12,7 +12,6 @@ role: eval: fleet: heavynode: - helixsensor: idh: import: manager: diff --git a/setup/so-functions b/setup/so-functions index 56c4b29c1..a6d8c585a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -562,7 +562,7 @@ check_requirements() { req_cores=4 req_nics=2 elif [[ $is_standalone ]]; then - req_mem=24 + req_mem=16 req_cores=4 req_nics=2 elif [[ $is_manager ]]; then @@ -586,7 +586,7 @@ check_requirements() { req_cores=4 req_nics=1 elif [[ $is_heavynode ]]; then - req_mem=24 + req_mem=16 req_cores=4 req_nics=2 elif [[ $is_idh ]]; then @@ -651,6 +651,17 @@ check_requirements() { if [[ $total_mem_hr -lt $req_mem ]]; then whiptail_requirements_error "memory" "${total_mem_hr} GB" "${req_mem} GB" + if [[ $is_standalone || $is_heavynode ]]; then + echo "This install type will fail with less than $req_mem GB of memory" + exit 0 + fi + fi + if [[ $is_standalone || $is_heavynode ]]; then + if [[ $total_mem_hr -gt 15 && $total_mem_hr -lt 24 ]]; then + low_mem=true + else + low_mem=false + fi fi } @@ -956,9 +967,7 @@ docker_seed_registry() { if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then if [ "$install_type" == 'IMPORT' ]; then - container_list 'so-import' - elif [ "$install_type" == 'HELIXSENSOR' ]; then - container_list 'so-helix' + container_list 'so-import' else container_list fi @@ -1217,7 +1226,7 @@ ls_heapsize() { fi case "$install_type" in - 'MANAGERSEARCH' | 'HEAVYNODE' | 'HELIXSENSOR' | 'STANDALONE') + 'MANAGERSEARCH' | 'HEAVYNODE' | 'STANDALONE') LS_HEAP_SIZE='1000m' ;; 'EVAL') @@ -1699,7 +1708,11 @@ drop_install_options() { echo "INTERFACE=$INTERFACE" >> /opt/so/install.txt NODETYPE=${install_type^^} echo "NODETYPE=$NODETYPE" >> /opt/so/install.txt - echo "CORECOUNT=$lb_procs" >> /opt/so/install.txt + if [[ $low_mem == "true" ]]; then + echo "CORECOUNT=1" >> /opt/so/install.txt + else + echo "CORECOUNT=$lb_procs" >> /opt/so/install.txt + fi echo "LSHOSTNAME=$HOSTNAME" >> /opt/so/install.txt echo "LSHEAP=$LS_HEAP_SIZE" >> /opt/so/install.txt echo "CPUCORES=$num_cpu_cores" >> /opt/so/install.txt From cc93976db99d82c9303386df1ebc1effc6bda2ee Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Nov 2023 15:17:23 -0400 Subject: [PATCH 53/78] Add memory restrictions --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index a6d8c585a..6ad0947d1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -652,7 +652,7 @@ check_requirements() { if [[ $total_mem_hr -lt $req_mem ]]; then whiptail_requirements_error "memory" "${total_mem_hr} GB" "${req_mem} GB" if [[ $is_standalone || $is_heavynode ]]; then - echo "This install type will fail with less than $req_mem GB of memory" + echo "This install type will fail with less than $req_mem GB of memory. Exiting setup." exit 0 fi fi From e6a0838e4cd2ac0bc8c42e686363a68d7f10d6f0 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Nov 2023 15:26:24 -0400 Subject: [PATCH 54/78] Add memory restrictions --- setup/so-setup | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 543ac0156..db5df492d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -676,7 +676,11 @@ if ! [[ -f $install_opt_file ]]; then export MAINIP=$MAINIP export PATCHSCHEDULENAME=$PATCHSCHEDULENAME export INTERFACE=$INTERFACE - export CORECOUNT=$lb_procs + if [[ $low_mem == "true" ]]; then + export CORECOUNT=1 + else + export CORECOUNT=$lb_procs + fi export LSHOSTNAME=$HOSTNAME export LSHEAP=$LS_HEAP_SIZE export CPUCORES=$num_cpu_cores From f33079f1e35792a31396f5064aa02dcbf690c2b7 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 20:09:56 +0000 Subject: [PATCH 55/78] Make settings global --- salt/elasticfleet/soc_elasticfleet.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 8cb975086..8685a96e5 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -43,32 +43,33 @@ elasticfleet: optional_integrations: sublime_platform: enabled_nodes: - description: Determines if the Sublime Platform integration is enabled. + description: Fleet nodes with the Sublime Platform integration enabled. Enter one per line. global: True helpLink: elastic-fleet.html advanced: True forcedType: "[]string" api_key: description: API key for Sublime Platform. - global: False + global: True helpLink: elastic-fleet.html advanced: True forcedType: string + sensitive: True base_url: description: Base URL for Sublime Platform. - global: False + global: True helpLink: elastic-fleet.html advanced: True forcedType: string poll_interval: description: Poll interval for alerts from Sublime Platform. - global: False + global: True helpLink: elastic-fleet.html advanced: True forcedType: string limit: description: The maximum number of message groups to return from Sublime Platform. - global: False + global: True helpLink: elastic-fleet.html advanced: True forcedType: int From 344dd7d61fac0c4802eed7baafa6f6c69ebb1381 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 1 Nov 2023 16:50:20 -0400 Subject: [PATCH 56/78] Add Elastic Fleet reset script --- .../tools/sbin_jinja/so-elastic-fleet-reset | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 salt/manager/tools/sbin_jinja/so-elastic-fleet-reset diff --git a/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset b/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset new file mode 100644 index 000000000..197ea0912 --- /dev/null +++ b/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. +{% from 'vars/globals.map.jinja' import GLOBALS %} + + +. /usr/sbin/so-common + +require_manager + + +# Inform user we are about to remove Elastic Fleet data +echo +echo "This script will remove the current Elastic Fleet install & all of its data and rerun Elastic Fleet setup." +echo +echo "If you would like to proceed, please type "AGREE" and hit ENTER." +echo +# Read user input +read INPUT +if [ "$INPUT" != "AGREE" ] ; then exit 0; fi + + +printf "\nUninstalling all Elastic Agents on all Grid Nodes...\n\n" +salt \* cmd.run "elastic-agent uninstall -f" queue=True + +printf "\nStopping Fleet Container...\n" +so-elastic-fleet-stop --force + +printf "\nDeleting Fleet Data from Pillars...\n" +sed -i -z "s/elasticfleet:.*grid_enrollment_heavy.*'//" /opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls +sed -i "/fleet_grid_enrollment_token_general.*/d" /opt/so/saltstack/local/pillar/global/soc_global.sls +sed -i "/fleet_grid_enrollment_token_heavy.*/d" /opt/so/saltstack/local/pillar/global/soc_global.sls + +printf "\n\nDeleting Elastic Fleet data...\n\n" + +ALIASES=".fleet-servers .fleet-policies-leader .fleet-agents .fleet-artifacts .fleet-enrollment-api-keys .kibana_ingest" +for ALIAS in ${ALIASES} +do + # Get all concrete indices from alias + INDXS=$(curl -K /opt/so/conf/kibana/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/_resolve/index/${ALIAS}" | jq -r '.aliases[].indices[]') + + # Delete all resolved indices + for INDX in ${INDXS} + do + printf "\nDeleting $INDX \n" + curl -K /opt/so/conf/kibana/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/${INDX}" -XDELETE + done +done + +printf "\n\nRestarting Kibana..\n" +so-kibana-restart --force + +wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "curl -K /opt/so/conf/elasticsearch/curl.config" + +printf "\nStarting Elastic Fleet Setup...\n" +so-elastic-fleet-setup + + +printf "\nRe-installing Elastic Agent on all Grid Nodes...\n\n" +salt \* state.apply elasticfleet.install_agent_grid queue=True + +printf "\nElastic Fleet Reset complete....\n" \ No newline at end of file From c230cf4eb7c197b554bd43dfbb8af11dd8144300 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 1 Nov 2023 17:00:32 -0400 Subject: [PATCH 57/78] Formatting --- salt/manager/tools/sbin_jinja/so-elastic-fleet-reset | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset b/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset index 197ea0912..35f867884 100644 --- a/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset +++ b/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset @@ -16,7 +16,7 @@ require_manager echo echo "This script will remove the current Elastic Fleet install & all of its data and rerun Elastic Fleet setup." echo -echo "If you would like to proceed, please type "AGREE" and hit ENTER." +echo "If you would like to proceed, please type AGREE and hit ENTER." echo # Read user input read INPUT @@ -50,7 +50,7 @@ do done done -printf "\n\nRestarting Kibana..\n" +printf "\n\nRestarting Kibana...\n" so-kibana-restart --force wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "curl -K /opt/so/conf/elasticsearch/curl.config" From c8d89971198b7afec72d60a56febc855781b1880 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Nov 2023 09:21:57 -0400 Subject: [PATCH 58/78] adjust log filter to include all hosts --- setup/so-verify | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-verify b/setup/so-verify index e9a8a375c..98bda96be 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -25,7 +25,8 @@ log_has_errors() { # Ignore salt mast cached public key and minion failed to auth because this is a test # to see if the salt key had already been accepted. - # Ignore failed to connect to ::1 since we have most curls wrapped in a retry. + # Ignore failed to connect to since we have most curls wrapped in a retry and there are + # multiple mirrors available. # Ignore perl-Error- since that is the name of a Perl package SO installs. @@ -39,7 +40,7 @@ log_has_errors() { grep -E "FAILED|Failed|failed|ERROR|Result: False|Error is not recoverable" "$setup_log" | \ grep -vE "The Salt Master has cached the public key for this node" | \ grep -vE "Minion failed to authenticate with the master" | \ - grep -vE "Failed to connect to ::1" | \ + grep -vE "Failed to connect to " | \ grep -vE "Failed to set locale" | \ grep -vE "perl-Error-" | \ grep -vE "Failed:\s*?[0-9]+" | \ From 65735fc4d3ff484eb02b0e43912373823267bb4d Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 2 Nov 2023 09:54:01 -0400 Subject: [PATCH 59/78] Add eval and import roles --- salt/elasticfleet/config.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index 02672d58f..3ff74efcd 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -102,7 +102,7 @@ eaoptionalintegrationsdir: {% for minion in node_data %} {% set role = node_data[minion]["role"] %} -{% if role in [ "fleet","heavynode", "manager","managersearch","standalone" ] %} +{% if role in [ "eval","fleet","heavynode","import","manager","managersearch","standalone" ] %} {% set optional_integrations = salt['pillar.get']('elasticfleet:optional_integrations', {}) %} {% set integration_keys = salt['pillar.get']('elasticfleet:optional_integrations', {}).keys() %} fleet_server_integrations_{{ minion }}: From 1d2518310dfdc59cc05a199f619fe6063f93623a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Nov 2023 09:59:45 -0400 Subject: [PATCH 60/78] more log bypass --- salt/common/tools/sbin/so-log-check | 1 + setup/so-verify | 3 +++ 2 files changed, 4 insertions(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 10f7e8c89..101001be0 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -160,6 +160,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|soc.field." # known ingest type collisions issue with earlier versions of SO EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error parsing signature" # Malformed Suricata rule, from upstream provider EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sticky buffer has no matches" # Non-critical Suricata error + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to determine destination index stats" # Elastic transform temporary error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed" diff --git a/setup/so-verify b/setup/so-verify index 98bda96be..e4d90b937 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -36,6 +36,8 @@ log_has_errors() { # This is ignored for Ubuntu # Failed to restart snapd.mounts-pre.target: Operation refused, unit snapd.mounts-pre.target # may be requested by dependency only (it is configured to refuse manual start/stop). + + # Exit code 100 failure is likely apt-get running in the background, we wait for it to unlock. grep -E "FAILED|Failed|failed|ERROR|Result: False|Error is not recoverable" "$setup_log" | \ grep -vE "The Salt Master has cached the public key for this node" | \ @@ -55,6 +57,7 @@ log_has_errors() { grep -vE "Login Failed Details" | \ grep -vE "response from daemon: unauthorized" | \ grep -vE "Reading first line of patchfile" | \ + grep -vE "Command failed with exit code 100; will retry" | \ grep -vE "Running scope as unit" &> "$error_log" if [[ $? -eq 0 ]]; then From 5388b92865bfc4c2903322ba5cadfcb54398932c Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 2 Nov 2023 10:20:32 -0400 Subject: [PATCH 61/78] Refactor & cleanup --- salt/common/tools/sbin/so-common | 4 ++ .../tools/sbin_jinja/so-elastic-fleet-setup | 11 +++++ .../tools/sbin_jinja/so-elastic-fleet-reset | 44 +++++++++++++------ setup/so-setup | 6 ++- 4 files changed, 49 insertions(+), 16 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index bfa61f1b7..8089db28b 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -551,6 +551,10 @@ set_version() { fi } +status () { + printf "\n=========================================================================\n$(date) | $1\n=========================================================================\n" +} + systemctl_func() { local action=$1 local echo_action=$1 diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 83a155ae6..d908d1df7 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -8,8 +8,19 @@ INTCA=/etc/pki/tls/certs/intca.crt +. /usr/sbin/so-common . /usr/sbin/so-elastic-fleet-common +# Check to make sure that Kibana API is up & ready +RETURN_CODE=0 +wait_for_web_response "http://localhost:5601/api/fleet/settings" "fleet" 300 "curl -K /opt/so/conf/elasticsearch/curl.config" +RETURN_CODE=$? + +if [[ "$RETURN_CODE" != "0" ]]; then + printf "Kibana API not accessible, exiting Elastic Fleet setup..." + exit 1 +fi + printf "\n### Create ES Token ###\n" ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq -r .value) diff --git a/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset b/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset index 35f867884..5a7be9e60 100644 --- a/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset +++ b/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset @@ -11,30 +11,39 @@ require_manager - # Inform user we are about to remove Elastic Fleet data echo echo "This script will remove the current Elastic Fleet install & all of its data and rerun Elastic Fleet setup." echo -echo "If you would like to proceed, please type AGREE and hit ENTER." +echo "If you would like to proceed, type AGREE and hit ENTER." echo # Read user input read INPUT -if [ "$INPUT" != "AGREE" ] ; then exit 0; fi +if [ "${INPUT^^}" != 'AGREE' ]; then exit 0; fi -printf "\nUninstalling all Elastic Agents on all Grid Nodes...\n\n" +status "Uninstalling all Elastic Agents on all Grid Nodes..." salt \* cmd.run "elastic-agent uninstall -f" queue=True -printf "\nStopping Fleet Container...\n" +status "Stopping Fleet Container..." so-elastic-fleet-stop --force -printf "\nDeleting Fleet Data from Pillars...\n" +status "Deleting Fleet Data from Pillars..." sed -i -z "s/elasticfleet:.*grid_enrollment_heavy.*'//" /opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls sed -i "/fleet_grid_enrollment_token_general.*/d" /opt/so/saltstack/local/pillar/global/soc_global.sls sed -i "/fleet_grid_enrollment_token_heavy.*/d" /opt/so/saltstack/local/pillar/global/soc_global.sls -printf "\n\nDeleting Elastic Fleet data...\n\n" +status "Deleting Elastic Fleet data..." + +# Check to make sure that Elasticsearch is up & ready +RETURN_CODE=0 +wait_for_web_response "https://localhost:9200/_cat/indices/.kibana*" "green open" 300 "curl -K /opt/so/conf/elasticsearch/curl.config" +RETURN_CODE=$? + +if [[ "$RETURN_CODE" != "0" ]]; then + status "Elasticsearch not accessible, exiting script..." + exit 1 +fi ALIASES=".fleet-servers .fleet-policies-leader .fleet-agents .fleet-artifacts .fleet-enrollment-api-keys .kibana_ingest" for ALIAS in ${ALIASES} @@ -45,21 +54,28 @@ do # Delete all resolved indices for INDX in ${INDXS} do - printf "\nDeleting $INDX \n" + status "Deleting $INDX" curl -K /opt/so/conf/kibana/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/${INDX}" -XDELETE done done -printf "\n\nRestarting Kibana...\n" +status "Restarting Kibana..." so-kibana-restart --force -wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "curl -K /opt/so/conf/elasticsearch/curl.config" +status "Checking to make sure that Kibana API is up & ready..." +RETURN_CODE=0 +wait_for_web_response "http://localhost:5601/api/fleet/settings" "fleet" 300 "curl -K /opt/so/conf/elasticsearch/curl.config" +RETURN_CODE=$? -printf "\nStarting Elastic Fleet Setup...\n" +if [[ "$RETURN_CODE" != "0" ]]; then + status "Kibana API not accessible, exiting script..." + exit 1 +fi + +status "Starting Elastic Fleet Setup..." so-elastic-fleet-setup - -printf "\nRe-installing Elastic Agent on all Grid Nodes...\n\n" +status "Re-installing Elastic Agent on all Grid Nodes..." salt \* state.apply elasticfleet.install_agent_grid queue=True -printf "\nElastic Fleet Reset complete....\n" \ No newline at end of file +status "Elastic Fleet Reset complete...." \ No newline at end of file diff --git a/setup/so-setup b/setup/so-setup index 543ac0156..2ea2809d5 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -768,8 +768,10 @@ if ! [[ -f $install_opt_file ]]; then info "Restarting SOC to pick up initial user" logCmd "so-soc-restart" title "Setting up Elastic Fleet" - logCmd "salt-call state.apply elasticfleet.config" - logCmd "so-elastic-fleet-setup" + logCmd "salt-call state.apply elasticfleet.config" + if ! logCmd so-setup-elastic-fleet; then + fail_setup + fi if [[ ! $is_import ]]; then title "Setting up Playbook" logCmd "so-playbook-reset" From 6c4dc7cc090d63faf93eeb952a1dcbdaf81c9c38 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 2 Nov 2023 10:23:03 -0400 Subject: [PATCH 62/78] fix UPGRADECOMMAND used for distrib salt upgrade. remove unneeded vars --- salt/salt/map.jinja | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 131ff46ca..5f687ef3f 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -5,22 +5,18 @@ {% set SPLITCHAR = '+' %} {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep -q salt ; echo $?', python_shell=True) %} {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %} - {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} - {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} {% set SYSTEMD_UNIT_FILE = '/lib/systemd/system/salt-minion.service' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %} {% set SALTPACKAGES = ['salt', 'salt-master', 'salt-minion'] %} - {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} - {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% set SYSTEMD_UNIT_FILE = '/usr/lib/systemd/system/salt-minion.service' %} {% endif %} {% set INSTALLEDSALTVERSION = grains.saltversion %} {% if grains.saltversion|string != SALTVERSION|string %} - {% if grains.os|lower in ['Rocky', 'redhat', 'CentOS Stream'] %} + {% if grains.os_family|lower == 'redhat' %} {% set UPGRADECOMMAND = 'yum clean all ; /usr/sbin/bootstrap-salt.sh -s 120 -r -F -x python3 stable ' ~ SALTVERSION %} {% elif grains.os_family|lower == 'debian' %} {% set UPGRADECOMMAND = '/usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION %} From 2c0e287f8c320f3d0f1d800a0916e90d59a93ca7 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 2 Nov 2023 10:34:24 -0400 Subject: [PATCH 63/78] Fix name --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 2ea2809d5..26955b893 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -769,7 +769,7 @@ if ! [[ -f $install_opt_file ]]; then logCmd "so-soc-restart" title "Setting up Elastic Fleet" logCmd "salt-call state.apply elasticfleet.config" - if ! logCmd so-setup-elastic-fleet; then + if ! logCmd so-elastic-fleet-setup; then fail_setup fi if [[ ! $is_import ]]; then From e18e0fd69a982d317a656a3d8e15d2bcd0160c8e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Nov 2023 10:39:14 -0400 Subject: [PATCH 64/78] more log bypass --- salt/common/tools/sbin/so-log-check | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 101001be0..0fd98a12e 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -161,6 +161,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error parsing signature" # Malformed Suricata rule, from upstream provider EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sticky buffer has no matches" # Non-critical Suricata error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to determine destination index stats" # Elastic transform temporary error + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeout retrieving docker" # Telegraf unable to reach Docker engine, rare EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed" From 32701b594187484a1944d6d4fc4072e822d5b468 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Nov 2023 12:50:12 -0400 Subject: [PATCH 65/78] more log bypass --- salt/common/tools/sbin/so-log-check | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 0fd98a12e..dc2b1d741 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -114,6 +114,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to poll" # server not yet ready (sensoroni waiting on soc) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|minions returned with non" # server not yet ready (salt waiting on minions) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so_long_term" # server not yet ready (influxdb not yet setup) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|search_phase_execution_exception" # server not yet ready (elastalert running searches before ES is ready) fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then From 51e7861757e04ecb2a66bfb3790ebcebdbe5a11d Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 2 Nov 2023 16:41:34 -0400 Subject: [PATCH 66/78] Don't source so-elastic-fleet-common if not there --- .../tools/sbin_jinja/so-elasticsearch-templates-load | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index aac6279fc..857da5434 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -9,7 +9,9 @@ . /usr/sbin/so-common {% if GLOBALS.role != 'so-heavynode' %} -. /usr/sbin/so-elastic-fleet-common +if [ -f /usr/sbin/so-elastic-fleet-common ]; then + . /usr/sbin/so-elastic-fleet-common +fi {% endif %} default_conf_dir=/opt/so/conf From 96fdfb382977de14430f18dcee522f441238b032 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Nov 2023 16:46:41 -0400 Subject: [PATCH 67/78] ignore connectivity problems to docker containers during startup --- salt/common/tools/sbin/so-log-check | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index dc2b1d741..170ef9506 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -115,6 +115,8 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|minions returned with non" # server not yet ready (salt waiting on minions) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so_long_term" # server not yet ready (influxdb not yet setup) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|search_phase_execution_exception" # server not yet ready (elastalert running searches before ES is ready) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeout retrieving docker" # Telegraf unable to reach Docker engine, rare + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeout retrieving container" # Telegraf unable to reach Docker engine, rare fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then @@ -162,7 +164,6 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error parsing signature" # Malformed Suricata rule, from upstream provider EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sticky buffer has no matches" # Non-critical Suricata error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to determine destination index stats" # Elastic transform temporary error - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeout retrieving docker" # Telegraf unable to reach Docker engine, rare EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed" From 3875970dc52de40d2e112082798f4285776383fd Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 2 Nov 2023 21:09:37 +0000 Subject: [PATCH 68/78] Add checkpoint and vsphere packages --- salt/elasticfleet/defaults.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index ba779f5a0..61ae5999f 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -35,6 +35,7 @@ elasticfleet: - azure - barracuda - carbonblack_edr + - checkpoint - cisco_asa - cisco_duo - cisco_meraki @@ -86,6 +87,7 @@ elasticfleet: - ti_otx - ti_recordedfuture - udp + - vsphere - windows - zscaler_zia - zscaler_zpa From 5bfef3f527bb530793ded8120fc91bdb728b03e3 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 2 Nov 2023 21:10:01 +0000 Subject: [PATCH 69/78] Add checkpoint and vsphere templates --- salt/elasticsearch/defaults.yaml | 220 +++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index cd8ff9397..02c2529a6 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1965,6 +1965,50 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-checkpoint_x_firewall: + index_sorting: False + index_template: + index_patterns: + - "logs-checkpoint.firewall-*" + template: + settings: + index: + lifecycle: + name: so-logs-checkpoint.firewall-logs + number_of_replicas: 0 + composed_of: + - "logs-checkpoint.firewall@package" + - "logs-checkpoint.firewall@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-cisco_asa_x_log: index_sorting: false index_template: @@ -7798,6 +7842,50 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-vsphere_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-vsphere.log-*" + template: + settings: + index: + lifecycle: + name: so-logs-vsphere.log-logs + number_of_replicas: 0 + composed_of: + - "logs-vsphere.log@package" + - "logs-vsphere.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-windows_x_forwarded: index_sorting: false index_template: @@ -8414,6 +8502,138 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-metrics-vsphere_x_datastore: + index_sorting: False + index_template: + index_patterns: + - "metrics-vsphere.datastore-*" + template: + settings: + index: + lifecycle: + name: so-metrics-vsphere.datastore-logs + number_of_replicas: 0 + composed_of: + - "metrics-vsphere.datastore@package" + - "metrics-vsphere.datastore@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-vsphere_x_host: + index_sorting: False + index_template: + index_patterns: + - "metrics-vsphere.host-*" + template: + settings: + index: + lifecycle: + name: so-metrics-vsphere.host-logs + number_of_replicas: 0 + composed_of: + - "metrics-vsphere.host@package" + - "metrics-vsphere.host@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-vsphere_x_virtualmachine: + index_sorting: False + index_template: + index_patterns: + - "metrics-vsphere.virtualmachine-*" + template: + settings: + index: + lifecycle: + name: so-metrics-vsphere.virtualmachine-logs + number_of_replicas: 0 + composed_of: + - "metrics-vsphere.virtualmachine@package" + - "metrics-vsphere.virtualmachine@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logstash: index_sorting: false index_template: From 8c7767b381462ae911abd7844e9261475c4b3bec Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 3 Nov 2023 08:41:33 -0400 Subject: [PATCH 70/78] Dont overwrite metadata --- .../pipelines/config/so/0012_input_elastic_agent.conf.jinja | 3 +++ .../pipelines/config/so/0013_input_lumberjack_fleet.conf | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja b/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja index 035436a96..6ba29f8e5 100644 --- a/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja +++ b/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja @@ -11,7 +11,10 @@ input { } } filter { +if ![metadata] { mutate { rename => {"@metadata" => "metadata"} } } +} + diff --git a/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf b/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf index 0377a81c4..fd9a87a22 100644 --- a/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf +++ b/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf @@ -13,10 +13,11 @@ input { filter { - if "fleet-lumberjack-input" in [tags] { +if ![metadata] { mutate { rename => {"@metadata" => "metadata"} } } } + From 1a3d4a2051bcd0f1379ccbb05145f6a4d519f7d2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 3 Nov 2023 09:14:26 -0400 Subject: [PATCH 71/78] ignore malformed open canary log lines --- salt/common/tools/sbin/so-log-check | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 170ef9506..d3aff6c14 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -140,6 +140,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example" # false positive (example test data) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200" # false positive (request successful, contained error string in content) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error" # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0 + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Canary running" # false positive (Open Canary logging out blank IP addresses) fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then From a8b0e41dbe70dfae77b2cfb116dc48df276cb819 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 3 Nov 2023 11:04:52 -0400 Subject: [PATCH 72/78] exit 0 --- salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index d908d1df7..9c06cb7c2 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -131,3 +131,4 @@ salt-call state.apply elasticfleet queue=True # Generate installers & install Elastic Agent on the node so-elastic-agent-gen-installers salt-call state.apply elasticfleet.install_agent_grid queue=True +exit 0 \ No newline at end of file From 3d8663db66e0f832fc022fb5e687107c72e14787 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 3 Nov 2023 11:29:45 -0400 Subject: [PATCH 73/78] Update soc_elasticsearch.yaml --- salt/elasticsearch/soc_elasticsearch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index a5170b776..ce795fe5a 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -463,7 +463,7 @@ elasticsearch: so-syslog: *indexSettings so-zeek: *indexSettings so_roles: - so-manger: &soroleSettings + so-manager: &soroleSettings config: node: roles: From 9d2b84818f2b7bb4bd20b601d52aead6653e072b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 3 Nov 2023 15:00:13 -0400 Subject: [PATCH 74/78] apply es and soc states to manager if new search or hn are added --- salt/manager/tools/sbin/so-minion | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index eca96da5c..1baf88cad 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -631,8 +631,15 @@ if [[ "$OPERATION" == 'add' || "$OPERATION" == 'setup' ]]; then # and they need to wait for ca creation to update the mine updateMine checkMine "network.ip_addrs" + # apply the elasticsearch state to the manager if a new searchnode was added + if [[ "$NODETYPE" == "SEARCHNODE" || "$NODETYPE" == "HEAVYNODE" ]]; then + # calls so-common and set_minionid sets MINIONID to local minion id + set_minionid + salt $MINIONID state.apply elasticsearch queue=True --async + salt $MINIONID state.apply soc queue=True --async + fi # run this async so the cli doesn't wait for a return - salt "$MINION_ID" state.highstate --async + salt "$MINION_ID" state.highstate --async queue=True fi fi From 0086c247296b226dde56c83271ac47664672bb89 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 3 Nov 2023 15:21:06 -0400 Subject: [PATCH 75/78] Upgrade Elastic Agent --- salt/manager/tools/sbin/soup | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 263fab7d0..27c6cb98d 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -431,8 +431,7 @@ post_to_2.4.4() { } post_to_2.4.5() { - echo "Regenerating Elastic Agent Installers" - /sbin/so-elastic-agent-gen-installers + echo "Nothing to apply" POSTVERSION=2.4.5 } @@ -449,7 +448,8 @@ post_to_2.4.20() { } post_to_2.4.30() { - echo "Nothing to apply" + echo "Regenerating Elastic Agent Installers" + /sbin/so-elastic-agent-gen-installers POSTVERSION=2.4.30 } @@ -511,7 +511,7 @@ up_to_2.4.4() { } up_to_2.4.5() { - determine_elastic_agent_upgrade + echo "Nothing to do for 2.4.5" INSTALLEDVERSION=2.4.5 } @@ -529,7 +529,7 @@ up_to_2.4.20() { } up_to_2.4.30() { - echo "Nothing to do for 2.4.30" + determine_elastic_agent_upgrade INSTALLEDVERSION=2.4.30 } From 7a0b21647f40f8217da0941917c3681237a210c8 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 4 Nov 2023 10:05:37 -0400 Subject: [PATCH 76/78] disregard false positives --- salt/common/tools/sbin/so-log-check | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index d3aff6c14..dc84ba5bd 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -117,6 +117,8 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|search_phase_execution_exception" # server not yet ready (elastalert running searches before ES is ready) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeout retrieving docker" # Telegraf unable to reach Docker engine, rare EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeout retrieving container" # Telegraf unable to reach Docker engine, rare + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error while communicating" # Elasticsearch MS -> HN "sensor" temporarily unavailable + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tls handshake error" # Docker registry container when new node comes onlines fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then @@ -140,7 +142,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example" # false positive (example test data) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200" # false positive (request successful, contained error string in content) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error" # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0 - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Canary running" # false positive (Open Canary logging out blank IP addresses) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal" # false positive (Open Canary logging out blank IP addresses) fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then From f6cd35e1435c49d1bca5078409ca3e92a1bb404f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 6 Nov 2023 08:03:31 -0500 Subject: [PATCH 77/78] Set execute permissions --- salt/elasticfleet/config.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index 3ff74efcd..5d5d3e826 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -68,6 +68,7 @@ eapackageupgrade: - source: salt://elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade - user: 947 - group: 939 + - mode: 755 - template: jinja {% if GLOBALS.role != "so-fleet" %} From b99c7ce76ee769635baeb6fb815430b1cad2c706 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 6 Nov 2023 11:22:35 -0500 Subject: [PATCH 78/78] improve verbosity of setup logs --- setup/so-setup | 1 + setup/so-verify | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 691d52a2c..a4e67535b 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -774,6 +774,7 @@ if ! [[ -f $install_opt_file ]]; then title "Setting up Elastic Fleet" logCmd "salt-call state.apply elasticfleet.config" if ! logCmd so-elastic-fleet-setup; then + error "Failed to run so-elastic-fleet-setup" fail_setup fi if [[ ! $is_import ]]; then diff --git a/setup/so-verify b/setup/so-verify index e4d90b937..3f00cc420 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -61,8 +61,11 @@ log_has_errors() { grep -vE "Running scope as unit" &> "$error_log" if [[ $? -eq 0 ]]; then + # This function succeeds (returns 0) if errors are detected return 0 fi + + # No errors found, return 1 (function failed to find errors) return 1 } @@ -121,7 +124,10 @@ main() { echo "WARNING: Failed setup a while ago" exit_code=1 elif log_has_errors; then - echo "WARNING: Errors detected during setup" + echo "WARNING: Errors detected during setup." + echo "--------- ERRORS ---------" + cat $error_log + echo "--------------------------" exit_code=1 touch /root/failure elif using_iso && cron_error_in_mail_spool; then