diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-lifecycle-status b/salt/common/tools/sbin/so-elasticsearch-ilm-lifecycle-status old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-delete b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-delete old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load old mode 100644 new mode 100755 index 09cc6dc08..dda521736 --- a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load @@ -6,68 +6,16 @@ . /usr/sbin/so-common +{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} +{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %} {%- set NODEIP = salt['pillar.get']('host:mainip', '') %} -# Set up ILM policies +{%- for index, settings in ES_INDEX_SETTINGS.items() %} + {%- if settings.policy is defined %} echo -echo "Setting up default Security Onion index lifecycle management policies..." - -# Elasticsearch logs +echo "Setting up {{ index }}-logs policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' echo -echo "Setting up Elasticsearch ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/elasticsearch-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Import logs -echo -echo "Setting up Import ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-import-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Kibana logs -echo -echo "Setting up Kibana ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-kibana-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Kratos logs -echo -echo "Setting up Kratos ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-kratos-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Logstash logs -echo -echo "Setting up Logstash ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-logstash-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Redis logs -echo -echo "Setting up Redis ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-redis-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Strelka logs -echo -echo "Setting up Strelka ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-strelka-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Suricata logs -echo -echo "Setting up Suricata ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-suricata-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Syslog logs -echo -echo "Setting up Syslog ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-syslog-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' -echo - -# Zeek logs -echo -echo "Setting up Zeek ILM policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/so-zeek-logs" -H 'Content-Type: application/json' -d'{ "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 }, "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } } } }' + {%- endif %} +{%- endfor %} echo diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-view b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-view old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-restart b/salt/common/tools/sbin/so-elasticsearch-ilm-restart old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-start b/salt/common/tools/sbin/so-elasticsearch-ilm-start old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-status b/salt/common/tools/sbin/so-elasticsearch-ilm-status old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-stop b/salt/common/tools/sbin/so-elasticsearch-ilm-stop old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 24772a479..d18a8a71f 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -25,7 +25,6 @@ container_list() { if [ $MANAGERCHECK == 'so-import' ]; then TRUSTED_CONTAINERS=( "so-elasticsearch" - "so-filebeat" "so-idstools" "so-influxdb" "so-kibana" @@ -47,7 +46,6 @@ container_list() { "so-elastic-agent" "so-elastic-agent-builder" "so-elasticsearch" - "so-filebeat" "so-idh" "so-idstools" "so-influxdb" @@ -72,7 +70,6 @@ container_list() { ) else TRUSTED_CONTAINERS=( - "so-filebeat" "so-idstools" "so-elasticsearch" "so-logstash" diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 3d1182255..a0c431881 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1430,6 +1430,8 @@ elasticsearch: date_detection: false settings: index: + lifecycle: + name: so-elasticsearch-logs mapping: total_fields: limit: 5000 @@ -1498,6 +1500,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-endgame: index_sorting: False index_template: @@ -2183,6 +2204,8 @@ elasticsearch: date_detection: false settings: index: + lifecycle: + name: so-suricata-logs mapping: total_fields: limit: 5000 @@ -2251,6 +2274,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-imperva: index_sorting: False index_template: @@ -2351,6 +2393,8 @@ elasticsearch: date_detection: false settings: index: + lifecycle: + name: so-import-logs mapping: total_fields: limit: 5000 @@ -2419,6 +2463,25 @@ elasticsearch: - common-dynamic-mappings - winlog-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-infoblox: index_sorting: False index_template: @@ -2671,6 +2734,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-kratos: warm: 7 close: 30 @@ -2754,6 +2836,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-logstash: index_sorting: False index_template: @@ -2770,6 +2871,8 @@ elasticsearch: date_detection: false settings: index: + lifecycle: + name: so-logstash-logs mapping: total_fields: limit: 5000 @@ -2838,6 +2941,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-microsoft: index_sorting: False index_template: @@ -3691,6 +3813,8 @@ elasticsearch: date_detection: false settings: index: + lifecycle: + name: so-redis-logs mapping: total_fields: limit: 5000 @@ -3759,6 +3883,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-snort: index_sorting: False index_template: @@ -4262,6 +4405,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-syslog: index_sorting: False index_template: @@ -4347,6 +4509,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-tomcat: index_sorting: False index_template: @@ -4447,6 +4628,8 @@ elasticsearch: date_detection: false settings: index: + lifecycle: + name: so-zeek-logs mapping: total_fields: limit: 5000 @@ -4517,6 +4700,25 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} so-zscaler: index_sorting: False index_template: diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 43e8d9f72..b9caa2e89 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -53,9 +53,19 @@ es_sync_scripts: - source: salt://elasticsearch/tools/sbin - exclude_pat: - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state + - so-elasticsearch-ilm-policy-load - defaults: GLOBALS: {{ GLOBALS }} +so-elasticsearch-ilm-policy-load-script: + file.managed: + - name: /usr/sbin/so-elasticsearch-ilm-policy-load + - source: salt://elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load + - user: 930 + - group: 939 + - mode: 754 + - template: jinja + so-elasticsearch-pipelines-script: file.managed: - name: /usr/sbin/so-elasticsearch-pipelines @@ -362,6 +372,16 @@ so-es-cluster-settings: - docker_container: so-elasticsearch - file: es_sync_scripts +so-elasticsearch-ilm-policy-load: + cmd.run: + - name: /usr/sbin/so-elasticsearch-ilm-policy-load + - cwd: /opt/so + - require: + - docker_container: so-elasticsearch + - file: so-elasticsearch-ilm-policy-load-script + - onchanges: + - file: so-elasticsearch-ilm-policy-load-script + so-elasticsearch-templates: cmd.run: - name: /usr/sbin/so-elasticsearch-templates-load diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index d9de9343e..26d4890c2 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -36,7 +36,7 @@ elasticsearch: global: True helpLink: elasticsearch.html index_settings: - so-aws: &indexSettings + so-elasticsearch: &indexSettings warm: description: Age (in days) of this index before it will move to warm storage, if warm nodes are present. Once moved, events on this index can take longer to fetch. global: True @@ -75,45 +75,51 @@ elasticsearch: description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. global: True helpLink: elasticsearch.html - so-azure: *indexSettings - so-barracuda: *indexSettings - so-beats: *indexSettings - so-bluecoat: *indexSettings - so-cef: *indexSettings - so-checkpoint: *indexSettings - so-cisco: *indexSettings - so-cyberark: *indexSettings - so-cylance: *indexSettings - so-elasticsearch: *indexSettings + policy: + phases: + hot: + min_age: + description: Minimum age + global: True + helpLink: elasticsearch.html + actions: + set_priority: + priority: + description: Priority of index, used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + global: True + helpLink: elasticsearch.html + rollover: + max_age: + description: Maximum age of index. Once an index reaches this limit, it will be rolled over into a new index. + global: True + helpLink: elasticsearch.html + max_primary_shard_size: + description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index. + global: True + helpLink: elasticsearch.html + cold: + min_age: + description: Minimum age of index, determining when it should be sent to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + global: True + helpLink: elasticsearch.html + actions: + set_priority: + priority: + description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + global: True + helpLink: elasticsearch.html + delete: + min_age: + description: Minimum age of index, determining when it should be deleted. + global: True + helpLink: elastic so-endgame: *indexSettings - so-f5: *indexSettings so-firewall: *indexSettings - so-fortinet: *indexSettings - so-gcp: *indexSettings - so-google_workspace: *indexSettings - so-ids: *indexSettings - so-imperva: *indexSettings so-import: *indexSettings - so-infoblox: *indexSettings - so-juniper: *indexSettings so-kibana: *indexSettings so-logstash: *indexSettings - so-microsoft: *indexSettings - so-misp: *indexSettings - so-netflow: *indexSettings - so-netscout: *indexSettings - so-o365: *indexSettings - so-okta: *indexSettings so-osquery: *indexSettings - so-proofpoint: *indexSettings - so-radware: *indexSettings so-redis: *indexSettings - so-snort: *indexSettings - so-snyk: *indexSettings - so-sonicwall: *indexSettings - so-sophos: *indexSettings so-strelka: *indexSettings so-syslog: *indexSettings - so-tomcat: *indexSettings so-zeek: *indexSettings - so-zscaler: *indexSettings diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load new file mode 100755 index 000000000..dda521736 --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load @@ -0,0 +1,21 @@ +#/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} +{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %} +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} + +{%- for index, settings in ES_INDEX_SETTINGS.items() %} + {%- if settings.policy is defined %} +echo +echo "Setting up {{ index }}-logs policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' +echo + {%- endif %} +{%- endfor %} +echo