From aa420b914b2bcc42f2d1d520a7b8b187e01dcc8f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 20 Aug 2021 16:27:09 -0400 Subject: [PATCH 001/136] Initial version so-playbook-import --- salt/common/tools/sbin/so-playbook-import | 22 ++++++++++++++++++++++ salt/soctopus/init.sls | 3 ++- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 salt/common/tools/sbin/so-playbook-import diff --git a/salt/common/tools/sbin/so-playbook-import b/salt/common/tools/sbin/so-playbook-import new file mode 100644 index 000000000..107851278 --- /dev/null +++ b/salt/common/tools/sbin/so-playbook-import @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +ENABLEPLAY=${1:-False} + +docker exec so-soctopus /usr/local/bin/python -c "import playbook; print(playbook.play_import($ENABLEPLAY))" diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls index c2c8dc1ac..724e5a617 100644 --- a/salt/soctopus/init.sls +++ b/salt/soctopus/init.sls @@ -10,7 +10,7 @@ soctopusdir: file.directory: - - name: /opt/so/conf/soctopus + - name: /opt/so/conf/soctopus/sigma-import - user: 939 - group: 939 - makedirs: True @@ -63,6 +63,7 @@ so-soctopus: - /opt/so/log/soctopus/:/var/log/SOCtopus/:rw - /opt/so/rules/elastalert/playbook:/etc/playbook-rules:rw - /opt/so/conf/navigator/nav_layer_playbook.json:/etc/playbook/nav_layer_playbook.json:rw + - /opt/so/conf/soctopus/sigma-import/:/SOCtopus/sigma-import/:rw {% if ISAIRGAP is sameas true %} - /nsm/repo/rules/sigma:/soctopus/sigma {% endif %} From b78da5c2377c4e1d282f74a01d47bbd43b35d5c0 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 23 Aug 2021 09:36:20 -0400 Subject: [PATCH 002/136] Merge hotfix to dev; reset to .80 --- HOTFIX | 2 -- VERSION | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/HOTFIX b/HOTFIX index 393a7e913..e69de29bb 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1,2 +0,0 @@ - -CURATOR diff --git a/VERSION b/VERSION index e183d6a6c..d5c0d128d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.70 +2.3.80 From 19cfce5e0bc83e41a1adbbaa313876e34df03d5e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 23 Aug 2021 10:47:41 -0400 Subject: [PATCH 003/136] Add curator delete yml files --- salt/curator/files/action/so-beats-delete.yml | 29 +++++++++++++++++++ .../files/action/so-firewall-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-ids-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-import-delete.yml | 29 +++++++++++++++++++ .../files/action/so-osquery-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-ossec-delete.yml | 29 +++++++++++++++++++ .../files/action/so-strelka-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-syslog-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-zeek-delete.yml | 29 +++++++++++++++++++ 9 files changed, 261 insertions(+) create mode 100644 salt/curator/files/action/so-beats-delete.yml create mode 100644 salt/curator/files/action/so-firewall-delete.yml create mode 100644 salt/curator/files/action/so-ids-delete.yml create mode 100644 salt/curator/files/action/so-import-delete.yml create mode 100644 salt/curator/files/action/so-osquery-delete.yml create mode 100644 salt/curator/files/action/so-ossec-delete.yml create mode 100644 salt/curator/files/action/so-strelka-delete.yml create mode 100644 salt/curator/files/action/so-syslog-delete.yml create mode 100644 salt/curator/files/action/so-zeek-delete.yml diff --git a/salt/curator/files/action/so-beats-delete.yml b/salt/curator/files/action/so-beats-delete.yml new file mode 100644 index 000000000..77931d661 --- /dev/null +++ b/salt/curator/files/action/so-beats-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-beats:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete beats indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-beats.*|so-beats.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-firewall-delete.yml b/salt/curator/files/action/so-firewall-delete.yml new file mode 100644 index 000000000..7588de437 --- /dev/null +++ b/salt/curator/files/action/so-firewall-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-firewall:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete firewall indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-firewall.*|so-firewall.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-ids-delete.yml b/salt/curator/files/action/so-ids-delete.yml new file mode 100644 index 000000000..e5bda4e34 --- /dev/null +++ b/salt/curator/files/action/so-ids-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-ids:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete IDS indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-import-delete.yml b/salt/curator/files/action/so-import-delete.yml new file mode 100644 index 000000000..aa9808c5f --- /dev/null +++ b/salt/curator/files/action/so-import-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-import:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-import.*|so-import.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-osquery-delete.yml b/salt/curator/files/action/so-osquery-delete.yml new file mode 100644 index 000000000..d77b1b3d1 --- /dev/null +++ b/salt/curator/files/action/so-osquery-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-osquery:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-osquery.*|so-osquery.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-ossec-delete.yml b/salt/curator/files/action/so-ossec-delete.yml new file mode 100644 index 000000000..7aea13e41 --- /dev/null +++ b/salt/curator/files/action/so-ossec-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-ossec:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete ossec indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ossec.*|so-ossec.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-strelka-delete.yml b/salt/curator/files/action/so-strelka-delete.yml new file mode 100644 index 000000000..3487aeb6d --- /dev/null +++ b/salt/curator/files/action/so-strelka-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-strelka:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete Strelka indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-strelka.*|so-strelka.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-syslog-delete.yml b/salt/curator/files/action/so-syslog-delete.yml new file mode 100644 index 000000000..5fe7417ad --- /dev/null +++ b/salt/curator/files/action/so-syslog-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-syslog:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete syslog indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-syslog.*|so-syslog.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-zeek-delete.yml b/salt/curator/files/action/so-zeek-delete.yml new file mode 100644 index 000000000..0694c2aed --- /dev/null +++ b/salt/curator/files/action/so-zeek-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete Zeek indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-zeek.*|so-zeek.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file From 6d49bca0ac4640a4251cc360625bd3a25bcd3f92 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 23 Aug 2021 15:36:11 -0400 Subject: [PATCH 004/136] Switch to new auth params --- salt/curator/files/curator.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml index 15ecb8cb1..5ec4bdc5f 100644 --- a/salt/curator/files/curator.yml +++ b/salt/curator/files/curator.yml @@ -19,7 +19,8 @@ client: - {{elasticsearch}} port: 9200 {%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} - http_auth: {{ ES_USER }}:{{ ES_PASS }} + username: {{ ES_USER }} + password: {{ ES_PASS }} {%- endif %} url_prefix: use_ssl: True From 9783d13ea3556b530a7cdb31ead3165d6175485d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 24 Aug 2021 10:22:01 -0400 Subject: [PATCH 005/136] remove identifier from HOTFIX file --- HOTFIX | 2 -- 1 file changed, 2 deletions(-) diff --git a/HOTFIX b/HOTFIX index 100732042..e69de29bb 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1,2 +0,0 @@ - -CURATOR GRAFANA_DASH_ALLOW From 0681d29bb085c60604ae023d3e0f71c6a165bd8c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 25 Aug 2021 10:23:06 -0400 Subject: [PATCH 006/136] starting es pillarization --- salt/elasticsearch/config.map.jinja | 5 + salt/elasticsearch/defaults.yaml | 1112 +++++++++++++++++++++++++++ salt/elasticsearch/init.sls | 11 + 3 files changed, 1128 insertions(+) create mode 100644 salt/elasticsearch/config.map.jinja create mode 100644 salt/elasticsearch/defaults.yaml diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja new file mode 100644 index 000000000..e6860e699 --- /dev/null +++ b/salt/elasticsearch/config.map.jinja @@ -0,0 +1,5 @@ +{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG %} + +{% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} + {% do ESCONFIG.elasticsearch.defaults.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} +{% endif %} diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml new file mode 100644 index 000000000..8fc244d6c --- /dev/null +++ b/salt/elasticsearch/defaults.yaml @@ -0,0 +1,1112 @@ +elasticsearch: + es_port: 9200 + esheap: 4049m + esclustername: default-cluster-name + log_size_limit: 95 #used for curator + + index_settings: + so-beats: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-firewall: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-flow: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-ids: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-import: + shards: 1 + warm: 7 + close: 73000 + delete: 73001 + so-osquery: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-ossec: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-strelka: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-syslog: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-zeek: + shards: 5 + warm: 7 + close: 45 + delete: 365 + + + + persistent: + cluster: + remote: + default-cluster-name: + seeds: + - 127.0.0.1:9300 + transient: + {} + defaults: + cluster: + max_voting_config_exclusions: 10 + auto_shrink_voting_configuration: true + election: + duration: 500ms + initial_timeout: 100ms + max_timeout: 10s + back_off_time: 100ms + strategy: supports_voting_only + no_master_block: write + persistent_tasks: + allocation: + enable: all + recheck_interval: 30s + blocks: + read_only_allow_delete: false + read_only: false + remote: + node: + attr: + initial_connect_timeout: 30s + connect: true + connections_per_cluster: 3 + follower_lag: + timeout: 90000ms + routing: + use_adaptive_replica_selection: true + rebalance: + enable: all + allocation: + node_concurrent_incoming_recoveries: 2 + include: + _tier: + node_initial_primaries_recoveries: 4 + same_shard: + host: false + total_shards_per_node: -1 + require: + _tier: + shard_state: + reroute: + priority: NORMAL + type: balanced + disk: + threshold_enabled: true + watermark: + flood_stage.frozen.max_headroom: 20GB + flood_stage: 98% + high: 98% + low: 95% + enable_for_single_data_node: false + flood_stage.frozen: 95% + include_relocations: true + reroute_interval: 60s + awareness: + attributes: [] + balance: + index: 0.55 + threshold: 1.0 + shard: 0.45 + enable: all + node_concurrent_outgoing_recoveries: 2 + allow_rebalance: indices_all_active + cluster_concurrent_rebalance: 2 + node_concurrent_recoveries: 2 + exclude: + _tier: + indices: + tombstones: + size: 500 + close: + enable: true + max_shards_per_node.frozen: 3000 + nodes: + reconnect_interval: 10s + service: + slow_master_task_logging_threshold: 10s + slow_task_logging_threshold: 30s + publish: + timeout: 30000ms + info_timeout: 10000ms + name: default-cluster-name + fault_detection: + leader_check: + interval: 1000ms + timeout: 10000ms + retry_count: 3 + follower_check: + interval: 1000ms + timeout: 10000ms + retry_count: 3 + join: + timeout: 60000ms + max_shards_per_node: 1000 + initial_master_nodes: [] + snapshot: + info: + max_concurrent_fetches: 5 + info: + update: + interval: 30s + timeout: 15s + stack: + templates: + enabled: true + logger: + level: INFO + bootstrap: + memory_lock: false + system_call_filter: true + ctrlhandler: true + processors: 8 + ingest: + user_agent: + cache_size: 1000 + geoip: + cache_size: 1000 + downloader: + enabled: false + endpoint: https://geoip.elastic.co/v1/database + poll: + interval: 3d + grok: + watchdog: + max_execution_time: 1s + interval: 1s + network: + host: + - 0.0.0.0 + tcp: + reuse_address: true + keep_count: -1 + connect_timeout: 30s + keep_interval: -1 + no_delay: true + keep_alive: true + receive_buffer_size: -1b + keep_idle: -1 + send_buffer_size: -1b + bind_host: + - 0.0.0.0 + server: true + breaker: + inflight_requests: + limit: 100% + overhead: 2.0 + publish_host: + - 0.0.0.0 + pidfile: + path: + data: [] + logs: /var/log/elasticsearch + shared_data: + home: /usr/share/elasticsearch + repo: [] + search: + default_search_timeout: -1 + highlight: + term_vector_multi_value: true + default_allow_partial_results: true + max_open_scroll_context: 500 + max_buckets: 65536 + low_level_cancellation: true + allow_expensive_queries: true + keep_alive_interval: 1m + remote: + node: + attr: + initial_connect_timeout: 30s + connect: true + connections_per_cluster: 3 + default_keep_alive: 5m + max_keep_alive: 24h + aggs: + rewrite_to_filter_by_filter: true + security: + manager: + filter_bad_defaults: true + transform: + task_thread_pool: + queue_size: 4 + size: 4 + ccr: + wait_for_metadata_timeout: 60s + indices: + recovery: + recovery_activity_timeout: 60s + chunk_size: 1mb + internal_action_timeout: 60s + max_bytes_per_sec: 40mb + max_concurrent_file_chunks: 5 + auto_follow: + wait_for_metadata_timeout: 60s + repositories: + fs: + compress: false + chunk_size: 9223372036854775807b + location: + url: + supported_protocols: + - http + - https + - ftp + - file + - jar + allowed_urls: [] + url: http: + action: + auto_create_index: true + search: + shard_count: + limit: 9223372036854775807 + destructive_requires_name: true + client: + type: node + transport: + ignore_cluster_name: false + nodes_sampler_interval: 5s + sniff: false + ping_timeout: 5s + enrich: + max_force_merge_attempts: 3 + cleanup_period: 15m + fetch_size: 10000 + coordinator_proxy: + max_concurrent_requests: 8 + max_lookups_per_request: 128 + queue_capacity: 1024 + max_concurrent_policy_executions: 50 + xpack: + flattened: + enabled: true + watcher: + execution: + scroll: + size: 0 + timeout: + default_throttle_period: 5s + internal: + ops: + bulk: + default_timeout: + index: + default_timeout: + search: + default_timeout: + thread_pool: + queue_size: 1000 + size: 40 + index: + rest: + direct_access: + use_ilm_index_management: true + history: + cleaner_service: + enabled: true + trigger: + schedule: + ticker: + tick_interval: 500ms + enabled: true + input: + search: + default_timeout: + encrypt_sensitive_data: false + transform: + search: + default_timeout: + stop: + timeout: 30s + watch: + scroll: + size: 0 + bulk: + concurrent_requests: 0 + flush_interval: 1s + size: 1mb + actions: 1 + actions: + bulk: + default_timeout: + index: + default_timeout: + eql: + enabled: true + data_frame: + enabled: true + ilm: + enabled: true + monitoring: + migration: + decommission_alerts: false + collection: + cluster: + stats: + timeout: 10s + node: + stats: + timeout: 10s + indices: [] + ccr: + stats: + timeout: 10s + enrich: + stats: + timeout: 10s + index: + stats: + timeout: 10s + recovery: + active_only: false + timeout: 10s + interval: 10s + enabled: false + ml: + job: + stats: + timeout: 10s + history: + duration: 168h + elasticsearch: + collection: + enabled: true + enabled: true + graph: + enabled: true + searchable: + snapshot: + allocate_on_rolling_restart: false + cache: + range_size: 32mb + sync: + max_files: 10000 + interval: 60s + shutdown_timeout: 10s + recovery_range_size: 128kb + shared_cache: + recovery_range_size: 128kb + region_size: 16mb + size: 0 + min_time_delta: 60s + decay: + interval: 60s + size.max_headroom: -1 + range_size: 16mb + max_freq: 100 + rollup: + enabled: true + task_thread_pool: + queue_size: -1 + size: 1 + sql: + enabled: true + searchable_snapshots: + cache_fetch_async_thread_pool: + core: 0 + max: 24 + keep_alive: 30s + cache_prewarming_thread_pool: + core: 0 + max: 16 + keep_alive: 30s + license: + upload: + types: + - standard + - gold + - platinum + - enterprise + - trial + self_generated: + type: basic + logstash: + enabled: true + notification: + pagerduty: + default_account: + email: + default_account: + html: + sanitization: + allow: + - body + - head + - _tables + - _links + - _blocks + - _formatting + - img:embedded + disallow: [] + enabled: true + reporting: + retries: 40 + warning: + enabled: true + interval: 15s + jira: + default_account: + slack: + default_account: + security: + operator_privileges: + enabled: false + dls_fls: + enabled: true + dls: + bitset: + cache: + size: 10% + ttl: 2h + transport: + filter: + allow: [] + deny: [] + enabled: true + ssl: + enabled: true + ssl: + diagnose: + trust: true + enabled: true + crypto: + thread_pool: + queue_size: 1000 + size: 4 + filter: + always_allow_bound_address: true + encryption: + algorithm: AES/CTR/NoPadding + audit: + enabled: false + logfile: + emit_node_id: true + emit_node_host_name: false + emit_node_name: false + events: + emit_request_body: false + include: + - ACCESS_DENIED + - ACCESS_GRANTED + - ANONYMOUS_ACCESS_DENIED + - AUTHENTICATION_FAILED + - CONNECTION_DENIED + - TAMPERED_REQUEST + - RUN_AS_DENIED + - RUN_AS_GRANTED + - SECURITY_CONFIG_CHANGE + exclude: + [] + emit_node_host_address: false + authc: + password_hashing: + algorithm: bcrypt + success_cache: + size: 10000 + enabled: true + expire_after_access: 1h + api_key: + doc_cache: + ttl: 5m + cache: + hash_algo: ssha256 + max_keys: 10000 + ttl: 24h + delete: + interval: 24h + timeout: -1 + enabled: false + hashing: + algorithm: pbkdf2 + anonymous: + authz_exception: true + roles: + - superuser + username: anonymous_user + run_as: + enabled: true + reserved_realm: + enabled: true + service_token: + cache: + hash_algo: ssha256 + max_tokens: 100000 + ttl: 20m + token: + delete: + interval: 30m + timeout: -1 + enabled: false + thread_pool: + queue_size: 1000 + size: 1 + timeout: 20m + fips_mode: + enabled: false + encryption_key: + length: 128 + algorithm: AES + http: + filter: + allow: [] + deny: [] + enabled: true + ssl: + enabled: true + automata: + max_determinized_states: 100000 + cache: + size: 10000 + ttl: 48h + enabled: true + user: null + authz: + store: + privileges: + cache: + ttl: 24h + max_size: 10000 + roles: + index: + cache: + ttl: 20m + max_size: 10000 + cache: + max_size: 10000 + negative_lookup_cache: + max_size: 10000 + field_permissions: + cache: + max_size_in_bytes: 104857600 + transform: + num_transform_failure_retries: 10 + enabled: true + vectors: + enabled: true + ccr: + enabled: true + ccr_thread_pool: + queue_size: 100 + size: 32 + idp: + privileges: + application: + cache: + size: 100 + ttl: 90m + metadata: + signing: + keystore: + alias: + slo_endpoint: + post: https: + redirect: https: + defaults: + nameid_format: urn:oasis:names:tc:SAML:2.0:nameid-format:transient + authn_expiry: 5m + allowed_nameid_formats: + - urn:oasis:names:tc:SAML:2.0:nameid-format:transient + contact: + given_name: + email: + surname: + organization: + display_name: + name: + url: http: + sso_endpoint: + post: https: + redirect: https: + entity_id: + signing: + keystore: + alias: + sp: + cache: + size: 1000 + ttl: 60m + wildcard: + path: wildcard_services.json + enabled: false + slm: + enabled: true + enrich: + enabled: true + http: + default_connection_timeout: 10s + proxy: + host: + scheme: + port: 0 + whitelist: + - * + default_read_timeout: 10s + max_response_size: 10mb + autoscaling: + memory: + monitor: + timeout: 15s + ml: + max_anomaly_records: 500 + enable_config_migration: true + max_open_jobs: 512 + min_disk_space_off_heap: 5gb + use_auto_machine_memory_percent: false + inference_model: + cache_size: 40% + time_to_live: 5m + nightly_maintenance_requests_per_second: -1.0 + node_concurrent_job_allocations: 2 + max_model_memory_limit: 0b + enabled: false + max_lazy_ml_nodes: 0 + max_ml_node_size: 0b + max_machine_memory_percent: 30 + persist_results_max_retries: 20 + autodetect_process: true + max_inference_processors: 50 + process_connect_timeout: 10s + rest: + action: + multi: + allow_explicit_index: true + cache: + recycler: + page: + limit: + heap: 10% + type: CONCURRENT + weight: + longs: 1.0 + ints: 1.0 + bytes: 1.0 + objects: 0.1 + async_search: + index_cleanup_interval: 1h + reindex: + remote: + whitelist: [] + resource: + reload: + enabled: true + interval: + low: 60s + high: 5s + medium: 30s + thread_pool: + force_merge: + queue_size: -1 + size: 1 + fetch_shard_started: + core: 1 + max: 16 + keep_alive: 5m + listener: + queue_size: -1 + size: 4 + refresh: + core: 1 + max: 4 + keep_alive: 5m + system_write: + queue_size: 1000 + size: 4 + generic: + core: 4 + max: 128 + keep_alive: 30s + warmer: + core: 1 + max: 4 + keep_alive: 5m + search: + max_queue_size: 1000 + queue_size: 1000 + size: 13 + auto_queue_frame_size: 2000 + target_response_time: 1s + min_queue_size: 1000 + fetch_shard_store: + core: 1 + max: 16 + keep_alive: 5m + flush: + core: 1 + max: 4 + keep_alive: 5m + management: + core: 1 + max: 5 + keep_alive: 5m + analyze: + queue_size: 16 + size: 1 + get: + queue_size: 1000 + size: 8 + system_read: + queue_size: 2000 + size: 4 + estimated_time_interval: 200ms + write: + queue_size: 10000 + size: 8 + snapshot: + core: 1 + max: 4 + keep_alive: 5m + search_throttled: + max_queue_size: 100 + queue_size: 100 + size: 1 + auto_queue_frame_size: 200 + target_response_time: 1s + min_queue_size: 100 + index: + codec: default + recovery: + type: + store: + type: + fs: + fs_lock: native + preload: [] + snapshot: + uncached_chunk_size: -1b + cache: + excluded_file_types: [] + monitor: + jvm: + gc: + enabled: true + overhead: + warn: 50 + debug: 10 + info: 25 + refresh_interval: 1s + refresh_interval: 1s + process: + refresh_interval: 1s + os: + refresh_interval: 1s + fs: + health: + enabled: true + refresh_interval: 120s + slow_path_logging_threshold: 5s + refresh_interval: 1s + runtime_fields: + grok: + watchdog: + max_execution_time: 1s + interval: 1s + transport: + tcp: + reuse_address: true + keep_count: -1 + connect_timeout: 30s + keep_interval: -1 + compress: false + port: 9300-9400 + no_delay: true + keep_alive: true + receive_buffer_size: -1b + keep_idle: -1 + send_buffer_size: -1b + bind_host: + - 0.0.0.0 + connect_timeout: 30s + compress: false + ping_schedule: -1 + connections_per_node: + recovery: 2 + state: 1 + bulk: 3 + reg: 6 + ping: 1 + tracer: + include: [] + exclude: + - internal:discovery/zen/fd* + - internal:coordination/fault_detection/* + - cluster:monitor/nodes/liveness + type: security4 + slow_operation_logging_threshold: 5s + type.default: netty4 + features: + x-pack: true + port: 9300-9400 + host: [] + publish_port: 9300 + tcp_no_delay: true + publish_host: {{ grains.host }} + netty: + receive_predictor_size: 64kb + receive_predictor_max: 64kb + worker_count: 8 + receive_predictor_min: 64kb + boss_count: 1 + script: + allowed_contexts: [] + max_compilations_rate: 20000/1m + cache: + max_size: 100 + expire: 0ms + painless: + regex: + enabled: limited + limit-factor: 6 + max_size_in_bytes: 65535 + allowed_types: [] + disable_max_compilations_rate: false + indexing_pressure: + memory: + limit: 10% + node: + data: true + roles: + - data_frozen + - data_warm + - transform + - data + - remote_cluster_client + - data_cold + - data_content + - data_hot + - ingest + - master + max_local_storage_nodes: 1 + processors: 8 + store: + allow_mmap: true + ingest: true + master: true + pidfile: + transform: true + remote_cluster_client: true + enable_lucene_segment_infos_trace: false + local_storage: true + name: {{ grains.host }} + id: + seed: 0 + voting_only: false + attr: + transform: + node: true + xpack: + installed: true + box_type: hot + portsfile: false + ml: true + indices: + replication: + retry_timeout: 60s + initial_retry_backoff_bound: 50ms + cache: + cleanup_interval: 1m + mapping: + dynamic_timeout: 30s + max_in_flight_updates: 10 + memory: + interval: 5s + max_index_buffer_size: -1 + shard_inactive_time: 5m + index_buffer_size: 10% + min_index_buffer_size: 48mb + breaker: + request: + limit: 60% + type: memory + overhead: 1.0 + total: + limit: 95% + use_real_memory: true + accounting: + limit: 100% + overhead: 1.0 + fielddata: + limit: 40% + type: memory + overhead: 1.03 + type: hierarchy + query: + bool: + max_nested_depth: 20 + max_clause_count: 1500 + query_string: + analyze_wildcard: false + allowLeadingWildcard: true + id_field_data: + enabled: true + recovery: + recovery_activity_timeout: 1800000ms + retry_delay_network: 5s + internal_action_timeout: 15m + retry_delay_state_sync: 500ms + internal_action_long_timeout: 1800000ms + max_concurrent_operations: 1 + max_bytes_per_sec: 40mb + max_concurrent_file_chunks: 2 + requests: + cache: + size: 1% + expire: 0ms + store: + delete: + shard: + timeout: 30s + analysis: + hunspell: + dictionary: + ignore_case: false + lazy: false + queries: + cache: + count: 10000 + size: 10% + all_segments: false + lifecycle: + history_index_enabled: true + poll_interval: 10m + step: + master_timeout: 30s + fielddata: + cache: + size: -1b + plugin: + mandatory: [] + slm: + minimum_interval: 15m + retention_schedule: 0 30 1 * * ? + retention_duration: 1h + history_index_enabled: true + discovery: + seed_hosts: [] + unconfigured_bootstrap_timeout: 3s + request_peers_timeout: 3000ms + zen: + commit_timeout: 30s + no_master_block: write + join_retry_delay: 100ms + join_retry_attempts: 3 + ping: + unicast: + concurrent_connects: 10 + hosts: [] + hosts.resolve_timeout: 5s + master_election: + ignore_non_master_pings: false + wait_for_joins_timeout: 30000ms + send_leave_request: true + ping_timeout: 3s + bwc_ping_timeout: 3s + join_timeout: 60000ms + publish_diff: + enable: true + publish: + max_pending_cluster_states: 25 + minimum_master_nodes: -1 + unsafe_rolling_upgrades_enabled: true + hosts_provider: [] + publish_timeout: 30s + fd: + connect_on_network_disconnect: false + ping_interval: 1s + ping_retries: 3 + register_connection_listener: true + ping_timeout: 30s + max_pings_from_another_master: 3 + initial_state_timeout: 30s + cluster_formation_warning_timeout: 10000ms + seed_providers: [] + type: single-node + seed_resolver: + max_concurrent_resolvers: 10 + timeout: 5s + find_peers_interval: 1000ms + probe: + connect_timeout: 30s + handshake_timeout: 30s + http: + cors: + max-age: 1728000 + allow-origin: + allow-headers: X-Requested-With,Content-Type,Content-Length + allow-credentials: false + allow-methods: OPTIONS,HEAD,GET,POST,PUT,DELETE + enabled: false + max_chunk_size: 8kb + compression_level: 3 + max_initial_line_length: 4kb + type: security4 + pipelining: + max_events: 10000 + type.default: netty4 + content_type: + required: true + host: [] + publish_port: -1 + read_timeout: 0ms + max_content_length: 100mb + netty: + receive_predictor_size: 64kb + max_composite_buffer_components: 69905 + worker_count: 0 + tcp: + reuse_address: true + keep_count: -1 + keep_interval: -1 + no_delay: true + keep_alive: true + receive_buffer_size: -1b + keep_idle: -1 + send_buffer_size: -1b + bind_host: [] + client_stats: + enabled: true + reset_cookies: false + max_warning_header_count: -1 + tracer: + include: [] + exclude: [] + max_warning_header_size: -1b + detailed_errors: + enabled: true + port: 9200-9300 + max_header_size: 8kb + tcp_no_delay: true + compression: false + publish_host: [] + gateway: + recover_after_master_nodes: 0 + expected_nodes: -1 + recover_after_data_nodes: -1 + expected_data_nodes: -1 + write_dangling_indices_info: true + slow_write_logging_threshold: 10s + recover_after_time: 0ms + expected_master_nodes: -1 + recover_after_nodes: -1 + auto_import_dangling_indices: false + snapshot: + refresh_repo_uuid_on_restore: true + max_concurrent_operations: 1000 diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 4045fa10f..09cba56f9 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -15,6 +15,8 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} + + {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} @@ -36,6 +38,7 @@ {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} +{% from 'elasticsearch/config.map.jinja' import ESCONFIG with context %} vm.max_map_count: @@ -142,6 +145,14 @@ esyml: - group: 939 - template: jinja +esyml_test: + file.managed: + - name: /tmp/elasticsearch.yml + - source: salt://elasticsearch/files/elasticsearch.yml + - user: 930 + - group: 939 + - contents: {{ ESCONFIG | yaml }} + #sync templates to /opt/so/conf/elasticsearch/templates {% for TEMPLATE in TEMPLATES %} es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: From fd1e6323861edaafb7ef965f7de02d4aac2e3fb9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 25 Aug 2021 12:08:43 -0400 Subject: [PATCH 007/136] cleanup yaml --- salt/elasticsearch/defaults.yaml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 8fc244d6c..7c506a9d2 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -273,7 +273,7 @@ elasticsearch: - file - jar allowed_urls: [] - url: http: + url: "http:" action: auto_create_index: true search: @@ -514,8 +514,7 @@ elasticsearch: - RUN_AS_DENIED - RUN_AS_GRANTED - SECURITY_CONFIG_CHANGE - exclude: - [] + exclude: [] emit_node_host_address: false authc: password_hashing: @@ -618,8 +617,8 @@ elasticsearch: keystore: alias: slo_endpoint: - post: https: - redirect: https: + post: "https:" + redirect: "https:" defaults: nameid_format: urn:oasis:names:tc:SAML:2.0:nameid-format:transient authn_expiry: 5m @@ -632,10 +631,10 @@ elasticsearch: organization: display_name: name: - url: http: + url: "http:" sso_endpoint: - post: https: - redirect: https: + post: "https:" + redirect: "https:" entity_id: signing: keystore: @@ -658,7 +657,7 @@ elasticsearch: scheme: port: 0 whitelist: - - * + - "*" default_read_timeout: 10s max_response_size: 10mb autoscaling: From a27569f20b16e9fe4c4a1b63b2108a9e52bfa059 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 25 Aug 2021 12:32:17 -0400 Subject: [PATCH 008/136] remove source when contents provided --- salt/elasticsearch/init.sls | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 09cba56f9..49f9fc04c 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -148,7 +148,6 @@ esyml: esyml_test: file.managed: - name: /tmp/elasticsearch.yml - - source: salt://elasticsearch/files/elasticsearch.yml - user: 930 - group: 939 - contents: {{ ESCONFIG | yaml }} From 7b93f355e21d23435ee8c9899b7b6ecd3e62612a Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 25 Aug 2021 15:17:19 -0400 Subject: [PATCH 009/136] so-import-evtx - timestamp extraction --- salt/common/tools/sbin/so-import-evtx | 175 +++++++++++++++++++++ salt/elasticsearch/files/ingest/import.wel | 6 + 2 files changed, 181 insertions(+) create mode 100644 salt/common/tools/sbin/so-import-evtx diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin/so-import-evtx new file mode 100644 index 000000000..fe53ee601 --- /dev/null +++ b/salt/common/tools/sbin/so-import-evtx @@ -0,0 +1,175 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +{%- set MANAGER = salt['grains.get']('master') %} +{%- set VERSION = salt['pillar.get']('global:soversion') %} +{%- set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} +{%- set MANAGERIP = salt['pillar.get']('global:managerip') -%} +{%- set URLBASE = salt['pillar.get']('global:url_base') %} +{% set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{% set ES_PW = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} + +INDEX_DATE=$(date +'%Y.%m.%d') +RUNID=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 8 | head -n 1) + +. /usr/sbin/so-common + +function usage { + cat << EOF +Usage: $0 [evtx-file-2] [evtx-file-*] + +Imports one or more evtx files into Security Onion. The evtx files will be analyzed and made available for review in the Security Onion toolset. +EOF +} + + +function evtx2es() { + EVTX=$1 + HASH=$2 + + docker run --rm \ + -v "$EVTX:/tmp/$RUNID.evtx" \ + so-evtx2es02 \ + --host {{ MANAGERIP }} --scheme https \ + --index so-beats-$INDEX_DATE --pipeline import.wel \ + --login {{ES_USER}} --pwd {{ES_PW}} "/tmp/$RUNID.evtx" 2>&1 + + + docker run --rm \ + -v "$EVTX:/tmp/import.evtx" \ + -v "/nsm/import/evtx-end_newest:/tmp/newest" \ + -v "/nsm/import/evtx-start_oldest:/tmp/oldest" \ + --entrypoint '/calculate_timestamp.sh' \ + so-evtx2es02 +} + +# if no parameters supplied, display usage +if [ $# -eq 0 ]; then + usage + exit 1 +fi + +# ensure this is a Manager node +require_manager + +# verify that all parameters are files +for i in "$@"; do + if ! [ -f "$i" ]; then + usage + echo "\"$i\" is not a valid file!" + exit 2 + fi +done + +# track if we have any valid or invalid pcaps +INVALID_EVTXS="no" +VALID_EVTXS="no" + +# track oldest start and newest end so that we can generate the Kibana search hyperlink at the end +START_OLDEST="2050-12-31" +END_NEWEST="1971-01-01" + +touch /nsm/import/evtx-start_oldest +touch /nsm/import/evtx-end_newest + +echo $START_OLDEST > /nsm/import/evtx-start_oldest +echo $END_NEWEST > /nsm/import/evtx-end_newest + +# paths must be quoted in case they include spaces +for EVTX in "$@"; do + EVTX=$(/usr/bin/realpath "$EVTX") + echo "Processing Import: ${EVTX}" + + # generate a unique hash to assist with dedupe checks + HASH=$(md5sum "${EVTX}" | awk '{ print $1 }') + HASH_DIR=/nsm/import/${HASH} + echo "- assigning unique identifier to import: $HASH" + + if [ -d $HASH_DIR ]; then + echo "- this EVTX has already been imported; skipping" + INVALID_EVTXS="yes" + else + VALID_EVTXS="yes" + + EVTX_DIR=$HASH_DIR/evtx + mkdir -p $EVTX_DIR + + # generate IDS alerts and write them to standard pipeline + echo "- importing logs with evtx2es" + evtx2es "${EVTX}" $HASH + + #START=$(pcapinfo "${EVTX}" -a |grep "First packet time:" | awk '{print $4}') + #END=$(pcapinfo "${EVTX}" -e |grep "Last packet time:" | awk '{print $4}') + #echo "- saving EVTX data spanning dates $START through $END" + + # compare $START to $START_OLDEST + START=$(cat /nsm/import/evtx-start_oldest) + START_COMPARE=$(date -d $START +%s) + START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s) + if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then + START_OLDEST=$START + fi + + # compare $ENDNEXT to $END_NEWEST + END=$(cat /nsm/import/evtx-end_newest) + ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"` + ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s) + END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s) + if [ $ENDNEXT_COMPARE -gt $END_NEWEST_COMPARE ]; then + END_NEWEST=$ENDNEXT + fi + + cp -f "${EVTX}" "${EVTX_DIR}"/data.evtx + chmod 644 "${EVTX_DIR}"/data.evtx + + fi # end of valid evtx + + echo + +done # end of for-loop processing evtx files + +# remove temp files +echo "Cleaning up:" +for TEMP_EVTX in ${TEMP_EVTXS[@]}; do + echo "- removing temporary evtx $TEMP_EVTX" + rm -f $TEMP_EVTX +done + +# output final messages +if [ "$INVALID_EVTXS" = "yes" ]; then + echo + echo "Please note! One or more evtx was invalid! You can scroll up to see which ones were invalid." +fi + +START_OLDEST_FORMATTED=`date +%Y-%m-%d --date="$START_OLDEST"` +START_OLDEST_SLASH=$(echo $START_OLDEST_FORMATTED | sed -e 's/-/%2F/g') +END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g') + +if [ "$VALID_EVTXS" = "yes" ]; then +cat << EOF + +Import complete! + +You can use the following hyperlink to view data in the time range of your import. You can triple-click to quickly highlight the entire hyperlink and you can then copy it into your browser: +https://{{ URLBASE }}/#/hunt?q=import.id:${RUNID}%20%7C%20groupby%20event.module%20event.dataset&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC + +or you can manually set your Time Range to be (in UTC): +From: $START_OLDEST_FORMATTED To: $END_NEWEST + +Please note that it may take 30 seconds or more for events to appear in Hunt. +EOF +fi diff --git a/salt/elasticsearch/files/ingest/import.wel b/salt/elasticsearch/files/ingest/import.wel index 5a04324b7..771d47b5d 100644 --- a/salt/elasticsearch/files/ingest/import.wel +++ b/salt/elasticsearch/files/ingest/import.wel @@ -4,6 +4,12 @@ { "remove": { "field": ["event.created","timestamp", "winlog.event_data.UtcTime", "event_record_id"], "ignore_failure": true } }, { "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } }, { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } }, + { + "dissect": { + "field": "log.file.name", + "pattern" : "/tmp/%{import.id}.evtx" + } +}, { "pipeline": { "name": "common" } } ] } \ No newline at end of file From de5f823abff6ac67e4fc6f50cede4ca44da08c84 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 25 Aug 2021 18:28:17 -0400 Subject: [PATCH 010/136] Add automation for deploy-vader env --- .../automation/vader-distributed-iso-manager | 76 ++++++++++++++++++ setup/automation/vader-distributed-iso-search | 78 ++++++++++++++++++ setup/automation/vader-distributed-iso-sensor | 80 +++++++++++++++++++ 3 files changed, 234 insertions(+) create mode 100644 setup/automation/vader-distributed-iso-manager create mode 100644 setup/automation/vader-distributed-iso-search create mode 100644 setup/automation/vader-distributed-iso-sensor diff --git a/setup/automation/vader-distributed-iso-manager b/setup/automation/vader-distributed-iso-manager new file mode 100644 index 000000000..07a22b588 --- /dev/null +++ b/setup/automation/vader-distributed-iso-manager @@ -0,0 +1,76 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +ALLOW_CIDR=0.0.0.0/0 +ALLOW_ROLE=a +BASICZEEK=7 +BASICSURI=7 +# BLOGS= +#BNICS=eth1 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=distributed-manager +install_type=MANAGER +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MANAGERADV=BASIC +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +# MSRV= +# MTU= +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +# REDIRECTHOST= +REDIRECTINFO=IP +RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r diff --git a/setup/automation/vader-distributed-iso-search b/setup/automation/vader-distributed-iso-search new file mode 100644 index 000000000..c1ced1537 --- /dev/null +++ b/setup/automation/vader-distributed-iso-search @@ -0,0 +1,78 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +# ALLOW_CIDR=0.0.0.0/0 +# ALLOW_ROLE=a +# BASICZEEK=7 +# BASICSURI=7 +# BLOGS= +# BNICS=eth1 +# ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +# GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=distributed-search +install_type=SEARCHNODE +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +# MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +MSRV=distributed-manager +MSRVIP=10.66.166.100 +# MTU= +# NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=ADVANCED +NODEUPDATES=MANAGER +# OINKCODE= +# OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +# PLAYBOOK=1 +# REDIRECTHOST= +# REDIRECTINFO=IP +# RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +# STRELKA=1 +# THEHIVE=1 +# WAZUH=1 +# WEBUSER=onionuser@somewhere.invalid +# WEBPASSWD1=0n10nus3r +# WEBPASSWD2=0n10nus3r diff --git a/setup/automation/vader-distributed-iso-sensor b/setup/automation/vader-distributed-iso-sensor new file mode 100644 index 000000000..356e67a2d --- /dev/null +++ b/setup/automation/vader-distributed-iso-sensor @@ -0,0 +1,80 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +# ALLOW_CIDR=0.0.0.0/0 +# ALLOW_ROLE=a +BASICZEEK=2 +BASICSURI=2 +# BLOGS= +BNICS=eth1 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +# GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=Distributed-Sensor +install_type=SENSOR +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +# MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +MSRV=distributed-manager +MSRVIP=10.66.166.100 +MTU=1500 +# NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +# NODESETUP=NODEBASIC +NSMSETUP=ADVANCED +NODEUPDATES=MANAGER +# OINKCODE= +# OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +# PLAYBOOK=1 +# REDIRECTHOST= +# REDIRECTINFO=IP +# RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +# STRELKA=1 +SURIPINS=(2 3) +# THEHIVE=1 +# WAZUH=1 +# WEBUSER=onionuser@somewhere.invalid +# WEBPASSWD1=0n10nus3r +# WEBPASSWD2=0n10nus3r +ZEEKPINS=(0 1) From 09d50978379ab7ee6a8cf2080cf6532772edb09e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 25 Aug 2021 21:08:49 -0400 Subject: [PATCH 011/136] Remove unused automation files --- .../automation/vader-distributed-iso-manager | 76 ------------------ setup/automation/vader-distributed-iso-search | 78 ------------------ setup/automation/vader-distributed-iso-sensor | 80 ------------------- 3 files changed, 234 deletions(-) delete mode 100644 setup/automation/vader-distributed-iso-manager delete mode 100644 setup/automation/vader-distributed-iso-search delete mode 100644 setup/automation/vader-distributed-iso-sensor diff --git a/setup/automation/vader-distributed-iso-manager b/setup/automation/vader-distributed-iso-manager deleted file mode 100644 index 07a22b588..000000000 --- a/setup/automation/vader-distributed-iso-manager +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -ALLOW_CIDR=0.0.0.0/0 -ALLOW_ROLE=a -BASICZEEK=7 -BASICSURI=7 -# BLOGS= -#BNICS=eth1 -ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=distributed-manager -install_type=MANAGER -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -MANAGERADV=BASIC -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=eth0 -# MSEARCH= -# MSRV= -# MTU= -NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -NODESETUP=NODEBASIC -NSMSETUP=BASIC -NODEUPDATES=MANAGER -# OINKCODE= -OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -PLAYBOOK=1 -# REDIRECTHOST= -REDIRECTINFO=IP -RULESETUP=ETOPEN -# SHARDCOUNT= -# SKIP_REBOOT= -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -STRELKA=1 -THEHIVE=1 -WAZUH=1 -WEBUSER=onionuser@somewhere.invalid -WEBPASSWD1=0n10nus3r -WEBPASSWD2=0n10nus3r diff --git a/setup/automation/vader-distributed-iso-search b/setup/automation/vader-distributed-iso-search deleted file mode 100644 index c1ced1537..000000000 --- a/setup/automation/vader-distributed-iso-search +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -# ALLOW_CIDR=0.0.0.0/0 -# ALLOW_ROLE=a -# BASICZEEK=7 -# BASICSURI=7 -# BLOGS= -# BNICS=eth1 -# ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -# GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=distributed-search -install_type=SEARCHNODE -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -# MANAGERADV=BASIC -MANAGERUPDATES=1 -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=eth0 -# MSEARCH= -MSRV=distributed-manager -MSRVIP=10.66.166.100 -# MTU= -# NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -NODESETUP=NODEBASIC -NSMSETUP=ADVANCED -NODEUPDATES=MANAGER -# OINKCODE= -# OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -# PLAYBOOK=1 -# REDIRECTHOST= -# REDIRECTINFO=IP -# RULESETUP=ETOPEN -# SHARDCOUNT= -# SKIP_REBOOT= -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -# STRELKA=1 -# THEHIVE=1 -# WAZUH=1 -# WEBUSER=onionuser@somewhere.invalid -# WEBPASSWD1=0n10nus3r -# WEBPASSWD2=0n10nus3r diff --git a/setup/automation/vader-distributed-iso-sensor b/setup/automation/vader-distributed-iso-sensor deleted file mode 100644 index 356e67a2d..000000000 --- a/setup/automation/vader-distributed-iso-sensor +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -# ALLOW_CIDR=0.0.0.0/0 -# ALLOW_ROLE=a -BASICZEEK=2 -BASICSURI=2 -# BLOGS= -BNICS=eth1 -ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -# GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=Distributed-Sensor -install_type=SENSOR -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -# MANAGERADV=BASIC -MANAGERUPDATES=1 -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=eth0 -# MSEARCH= -MSRV=distributed-manager -MSRVIP=10.66.166.100 -MTU=1500 -# NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -# NODESETUP=NODEBASIC -NSMSETUP=ADVANCED -NODEUPDATES=MANAGER -# OINKCODE= -# OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -# PLAYBOOK=1 -# REDIRECTHOST= -# REDIRECTINFO=IP -# RULESETUP=ETOPEN -# SHARDCOUNT= -# SKIP_REBOOT= -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -# STRELKA=1 -SURIPINS=(2 3) -# THEHIVE=1 -# WAZUH=1 -# WEBUSER=onionuser@somewhere.invalid -# WEBPASSWD1=0n10nus3r -# WEBPASSWD2=0n10nus3r -ZEEKPINS=(0 1) From c5d120293df57e96af46195f33d9c4a4f3630c24 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 27 Aug 2021 11:33:51 -0400 Subject: [PATCH 012/136] Initial work to add unattended option to soup --- salt/common/tools/sbin/soup | 91 +++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 33 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index ce8923e90..8adabf908 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -105,17 +105,20 @@ add_common() { airgap_mounted() { # Let's see if the ISO is already mounted. - if [ -f /tmp/soagupdate/SecurityOnion/VERSION ]; then + if [[ -f /tmp/soagupdate/SecurityOnion/VERSION ]]; then echo "The ISO is already mounted" else - echo "" - cat << EOF + if [[ -z $ISOLOC ]]; then + echo "This is airgap. Ask for a location." + echo "" + cat << EOF In order for soup to proceed, the path to the downloaded Security Onion ISO file, or the path to the CD-ROM or equivalent device containing the ISO media must be provided. For example, if you have copied the new Security Onion ISO file to your home directory, then the path might look like /home/myuser/securityonion-2.x.y.iso. Or, if you have burned the new ISO onto an optical disk then the path might look like /dev/cdrom. EOF - read -rp 'Enter the path to the new Security Onion ISO content: ' ISOLOC + read -rp 'Enter the path to the new Security Onion ISO content: ' ISOLOC + fi if [[ -f $ISOLOC ]]; then # Mounting the ISO image mkdir -p /tmp/soagupdate @@ -124,23 +127,27 @@ EOF if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then echo "Something went wrong trying to mount the ISO." echo "Ensure you verify the ISO that you downloaded." - exit 0 + exit 1 else echo "ISO has been mounted!" fi elif [[ -f $ISOLOC/SecurityOnion/VERSION ]]; then ln -s $ISOLOC /tmp/soagupdate echo "Found the update content" - else + elif [[ -b $ISOLOC ]]; then mkdir -p /tmp/soagupdate mount $ISOLOC /tmp/soagupdate if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then echo "Something went wrong trying to mount the device." echo "Ensure you verify the ISO that you downloaded." - exit 0 + exit 1 else echo "Device has been mounted!" - fi + fi + else + echo "Could not find Security Onion ISO content at ${ISOLOC}" + echo "Ensure the path you entered is correct, and that you verify the ISO that you downloaded." + exit 1 fi fi } @@ -774,39 +781,22 @@ verify_latest_update_script() { } main() { - set -e - set +e trap 'check_err $?' EXIT - echo "### Preparing soup at $(date) ###" - while getopts ":b" opt; do - case "$opt" in - b ) # process option b - shift - BATCHSIZE=$1 - if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then - echo "Batch size must be a number greater than 0." - exit 1 - fi - ;; - \? ) - echo "Usage: cmd [-b]" - ;; - esac - done - + echo "Checking to see if this is an airgap install." + echo "" + check_airgap + if [[ $is_airgap -eq 0 && $UNATTENDED == true && -z $ISOLOC ]]; then + echo "Missing file argument for unattended airgap upgrade." + fi echo "Checking to see if this is a manager." echo "" require_manager set_minionid - echo "Checking to see if this is an airgap install." - echo "" - check_airgap echo "Found that Security Onion $INSTALLEDVERSION is currently installed." echo "" if [[ $is_airgap -eq 0 ]]; then # Let's mount the ISO since this is airgap - echo "This is airgap. Ask for a location." airgap_mounted else echo "Cloning Security Onion github repo into $UPDATE_DIR." @@ -1029,7 +1019,40 @@ EOF echo "### soup has been served at $(date) ###" } -cat << EOF +while getopts ":b:f:y" opt; do + case ${opt} in + b ) + BATCHSIZE="$OPTARG" + if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then + echo "Batch size must be a number greater than 0." + exit 1 + fi + ;; + y ) + if [[ ! -f /opt/so/state/yeselastic.txt ]]; then + echo "Cannot run soup in unattended mode. You must run soup manually to accept the Elastic License." + exit 1 + else + UNATTENDED=true + fi + ;; + f ) + ISOLOC="$OPTARG" + ;; + \? ) + echo "Usage: soup [-b] [-y] [-f ]" + exit 1 + ;; + : ) + echo "Invalid option: $OPTARG requires an argument" + exit 1 + ;; + esac +done +shift $((OPTIND - 1)) + +if [[ -z $UNATTENDED ]]; then + cat << EOF SOUP - Security Onion UPdater @@ -1041,7 +1064,9 @@ Press Enter to continue or Ctrl-C to cancel. EOF -read -r input + read -r input +fi +echo "### Preparing soup at $(date) ###" main "$@" | tee -a $SOUP_LOG From ca5339341f6f39fc91f6d3e6fb089d4e71c4b5a4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 27 Aug 2021 11:34:28 -0400 Subject: [PATCH 013/136] Fix batch size regex to disallow 0 --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 8adabf908..60e0ce93f 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -1023,7 +1023,7 @@ while getopts ":b:f:y" opt; do case ${opt} in b ) BATCHSIZE="$OPTARG" - if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then + if ! [[ "$BATCHSIZE" =~ ^[1-9][0-9]*$ ]]; then echo "Batch size must be a number greater than 0." exit 1 fi From f8cdf5bca3987112ceb13305a124a5a2ae6a880a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 27 Aug 2021 14:39:02 -0400 Subject: [PATCH 014/136] Add Templates for all filebeat modules --- .../templates/so/so-aws-template.json.jinja | 13 +++++++++++++ .../templates/so/so-azure-template.json.jinja | 13 +++++++++++++ .../templates/so/so-barracuda-template.json.jinja | 13 +++++++++++++ .../templates/so/so-bluecoat-template.json.jinja | 13 +++++++++++++ .../templates/so/so-cef-template.json.jinja | 13 +++++++++++++ .../templates/so/so-checkpoint-template.json.jinja | 13 +++++++++++++ .../templates/so/so-cisco-template.json.jinja | 13 +++++++++++++ .../templates/so/so-cyberark-template.json.jinja | 13 +++++++++++++ .../templates/so/so-cylance-template.json.jinja | 13 +++++++++++++ .../so/so-elasticsearch-template.json.jinja | 13 +++++++++++++ .../templates/so/so-f5-template.json.jinja | 13 +++++++++++++ .../templates/so/so-fortinet-template.json.jinja | 13 +++++++++++++ .../templates/so/so-gcp-template.json.jinja | 13 +++++++++++++ .../so/so-google_workspace-template.json.jinja | 13 +++++++++++++ .../templates/so/so-imperva-template.json.jinja | 13 +++++++++++++ .../templates/so/so-infoblox-template.json.jinja | 13 +++++++++++++ .../templates/so/so-juniper-template.json.jinja | 13 +++++++++++++ .../templates/so/so-kibana-template.json.jinja | 13 +++++++++++++ .../templates/so/so-logstash-template.json.jinja | 13 +++++++++++++ .../templates/so/so-microsoft-template.json.jinja | 13 +++++++++++++ .../templates/so/so-misp-template.json.jinja | 13 +++++++++++++ .../templates/so/so-netflow-template.json.jinja | 13 +++++++++++++ .../templates/so/so-netscout-template.json.jinja | 13 +++++++++++++ .../templates/so/so-o365-template.json.jinja | 13 +++++++++++++ .../templates/so/so-okta-template.json.jinja | 13 +++++++++++++ .../templates/so/so-proofpoint-template.json.jinja | 13 +++++++++++++ .../templates/so/so-radware-template.json.jinja | 13 +++++++++++++ .../templates/so/so-redis-template.json.jinja | 13 +++++++++++++ .../templates/so/so-snort-template.json.jinja | 13 +++++++++++++ .../templates/so/so-snyk-template.json.jinja | 13 +++++++++++++ .../templates/so/so-sonicwall-template.json.jinja | 13 +++++++++++++ .../templates/so/so-sophos-template.json.jinja | 13 +++++++++++++ .../templates/so/so-squid-template.json.jinja | 13 +++++++++++++ .../templates/so/so-tomcat-template.json.jinja | 13 +++++++++++++ .../templates/so/so-zscaler-template.json.jinja | 13 +++++++++++++ 35 files changed, 455 insertions(+) create mode 100644 salt/elasticsearch/templates/so/so-aws-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-azure-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-barracuda-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-bluecoat-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-cef-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-checkpoint-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-cisco-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-cyberark-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-cylance-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-elasticsearch-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-f5-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-fortinet-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-gcp-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-google_workspace-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-imperva-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-infoblox-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-juniper-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-kibana-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-logstash-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-microsoft-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-misp-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-netflow-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-netscout-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-o365-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-okta-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-proofpoint-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-radware-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-redis-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-snort-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-snyk-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-sonicwall-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-sophos-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-squid-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-tomcat-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-zscaler-template.json.jinja diff --git a/salt/elasticsearch/templates/so/so-aws-template.json.jinja b/salt/elasticsearch/templates/so/so-aws-template.json.jinja new file mode 100644 index 000000000..19b23dfba --- /dev/null +++ b/salt/elasticsearch/templates/so/so-aws-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-aws:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-aws:refresh', '30s') %} +{ + "index_patterns": ["so-aws-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-azure-template.json.jinja b/salt/elasticsearch/templates/so/so-azure-template.json.jinja new file mode 100644 index 000000000..51a266479 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-azure-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-azure:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-azure:refresh', '30s') %} +{ + "index_patterns": ["so-azure-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-barracuda-template.json.jinja b/salt/elasticsearch/templates/so/so-barracuda-template.json.jinja new file mode 100644 index 000000000..66967d6d1 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-barracuda-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-barracuda:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-barracuda:refresh', '30s') %} +{ + "index_patterns": ["so-barracuda-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-bluecoat-template.json.jinja b/salt/elasticsearch/templates/so/so-bluecoat-template.json.jinja new file mode 100644 index 000000000..b1714183e --- /dev/null +++ b/salt/elasticsearch/templates/so/so-bluecoat-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-bluecoat:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-bluecoat:refresh', '30s') %} +{ + "index_patterns": ["so-bluecoat-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-cef-template.json.jinja b/salt/elasticsearch/templates/so/so-cef-template.json.jinja new file mode 100644 index 000000000..0081d42e1 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-cef-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-cef:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-cef:refresh', '30s') %} +{ + "index_patterns": ["so-cef-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-checkpoint-template.json.jinja b/salt/elasticsearch/templates/so/so-checkpoint-template.json.jinja new file mode 100644 index 000000000..5d41946cf --- /dev/null +++ b/salt/elasticsearch/templates/so/so-checkpoint-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-checkpoint:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-checkpoint:refresh', '30s') %} +{ + "index_patterns": ["so-checkpoint-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-cisco-template.json.jinja b/salt/elasticsearch/templates/so/so-cisco-template.json.jinja new file mode 100644 index 000000000..e6e6d14d0 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-cisco-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-cisco:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-cisco:refresh', '30s') %} +{ + "index_patterns": ["so-cisco-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-cyberark-template.json.jinja b/salt/elasticsearch/templates/so/so-cyberark-template.json.jinja new file mode 100644 index 000000000..1647d600f --- /dev/null +++ b/salt/elasticsearch/templates/so/so-cyberark-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-cyberark:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-cyberark:refresh', '30s') %} +{ + "index_patterns": ["so-cyberark-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-cylance-template.json.jinja b/salt/elasticsearch/templates/so/so-cylance-template.json.jinja new file mode 100644 index 000000000..4ba7d0316 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-cylance-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-cylance:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-cylance:refresh', '30s') %} +{ + "index_patterns": ["so-cylance-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-elasticsearch-template.json.jinja b/salt/elasticsearch/templates/so/so-elasticsearch-template.json.jinja new file mode 100644 index 000000000..16aaaec13 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-elasticsearch-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-elasticsearch:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-elasticsearch:refresh', '30s') %} +{ + "index_patterns": ["so-elasticsearch-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-f5-template.json.jinja b/salt/elasticsearch/templates/so/so-f5-template.json.jinja new file mode 100644 index 000000000..682a37c59 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-f5-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-f5:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-f5:refresh', '30s') %} +{ + "index_patterns": ["so-f5-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-fortinet-template.json.jinja b/salt/elasticsearch/templates/so/so-fortinet-template.json.jinja new file mode 100644 index 000000000..616607f52 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-fortinet-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-gcp-template.json.jinja b/salt/elasticsearch/templates/so/so-gcp-template.json.jinja new file mode 100644 index 000000000..4f1db4f20 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-gcp-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-gcp:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-gcp:refresh', '30s') %} +{ + "index_patterns": ["so-gcp-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-google_workspace-template.json.jinja b/salt/elasticsearch/templates/so/so-google_workspace-template.json.jinja new file mode 100644 index 000000000..5ae26780a --- /dev/null +++ b/salt/elasticsearch/templates/so/so-google_workspace-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-google_workspace:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-google_workspace:refresh', '30s') %} +{ + "index_patterns": ["so-google_workspace-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-imperva-template.json.jinja b/salt/elasticsearch/templates/so/so-imperva-template.json.jinja new file mode 100644 index 000000000..1f574f33a --- /dev/null +++ b/salt/elasticsearch/templates/so/so-imperva-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-imperva:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-imperva:refresh', '30s') %} +{ + "index_patterns": ["so-imperva-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-infoblox-template.json.jinja b/salt/elasticsearch/templates/so/so-infoblox-template.json.jinja new file mode 100644 index 000000000..de613de7f --- /dev/null +++ b/salt/elasticsearch/templates/so/so-infoblox-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-infoblox:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-infoblox:refresh', '30s') %} +{ + "index_patterns": ["so-infoblox-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-juniper-template.json.jinja b/salt/elasticsearch/templates/so/so-juniper-template.json.jinja new file mode 100644 index 000000000..f637271a9 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-juniper-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-juniper:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-juniper:refresh', '30s') %} +{ + "index_patterns": ["so-juniper-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-kibana-template.json.jinja b/salt/elasticsearch/templates/so/so-kibana-template.json.jinja new file mode 100644 index 000000000..fe2004b0e --- /dev/null +++ b/salt/elasticsearch/templates/so/so-kibana-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-kibana:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-kibana:refresh', '30s') %} +{ + "index_patterns": ["so-kibana-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-logstash-template.json.jinja b/salt/elasticsearch/templates/so/so-logstash-template.json.jinja new file mode 100644 index 000000000..2cf0aba42 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-logstash-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-logstash:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-logstash:refresh', '30s') %} +{ + "index_patterns": ["so-logstash-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-microsoft-template.json.jinja b/salt/elasticsearch/templates/so/so-microsoft-template.json.jinja new file mode 100644 index 000000000..3493ccbb2 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-microsoft-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-microsoft:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-microsoft:refresh', '30s') %} +{ + "index_patterns": ["so-microsoft-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-misp-template.json.jinja b/salt/elasticsearch/templates/so/so-misp-template.json.jinja new file mode 100644 index 000000000..67af1efde --- /dev/null +++ b/salt/elasticsearch/templates/so/so-misp-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-misp:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-misp:refresh', '30s') %} +{ + "index_patterns": ["so-misp-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-netflow-template.json.jinja b/salt/elasticsearch/templates/so/so-netflow-template.json.jinja new file mode 100644 index 000000000..62c0972bf --- /dev/null +++ b/salt/elasticsearch/templates/so/so-netflow-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-netflow:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-netflow:refresh', '30s') %} +{ + "index_patterns": ["so-netflow-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-netscout-template.json.jinja b/salt/elasticsearch/templates/so/so-netscout-template.json.jinja new file mode 100644 index 000000000..1dfe336d9 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-netscout-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-netscout:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-netscout:refresh', '30s') %} +{ + "index_patterns": ["so-netscout-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-o365-template.json.jinja b/salt/elasticsearch/templates/so/so-o365-template.json.jinja new file mode 100644 index 000000000..c1f4826f4 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-o365-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-o365:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-o365:refresh', '30s') %} +{ + "index_patterns": ["so-o365-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-okta-template.json.jinja b/salt/elasticsearch/templates/so/so-okta-template.json.jinja new file mode 100644 index 000000000..a4f2df44e --- /dev/null +++ b/salt/elasticsearch/templates/so/so-okta-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-okta:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-okta:refresh', '30s') %} +{ + "index_patterns": ["so-okta-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-proofpoint-template.json.jinja b/salt/elasticsearch/templates/so/so-proofpoint-template.json.jinja new file mode 100644 index 000000000..d7b0ecbe2 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-proofpoint-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-proofpoint:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-proofpoint:refresh', '30s') %} +{ + "index_patterns": ["so-proofpoint-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-radware-template.json.jinja b/salt/elasticsearch/templates/so/so-radware-template.json.jinja new file mode 100644 index 000000000..4efef6a4f --- /dev/null +++ b/salt/elasticsearch/templates/so/so-radware-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-radware:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-radware:refresh', '30s') %} +{ + "index_patterns": ["so-radware-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-redis-template.json.jinja b/salt/elasticsearch/templates/so/so-redis-template.json.jinja new file mode 100644 index 000000000..616607f52 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-redis-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-snort-template.json.jinja b/salt/elasticsearch/templates/so/so-snort-template.json.jinja new file mode 100644 index 000000000..325f86bde --- /dev/null +++ b/salt/elasticsearch/templates/so/so-snort-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-snort:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-snort:refresh', '30s') %} +{ + "index_patterns": ["so-snort-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-snyk-template.json.jinja b/salt/elasticsearch/templates/so/so-snyk-template.json.jinja new file mode 100644 index 000000000..616607f52 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-snyk-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-sonicwall-template.json.jinja b/salt/elasticsearch/templates/so/so-sonicwall-template.json.jinja new file mode 100644 index 000000000..b912e4d66 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-sonicwall-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-sonicwall:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-sonicwall:refresh', '30s') %} +{ + "index_patterns": ["so-sonicwall-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-sophos-template.json.jinja b/salt/elasticsearch/templates/so/so-sophos-template.json.jinja new file mode 100644 index 000000000..616607f52 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-sophos-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-squid-template.json.jinja b/salt/elasticsearch/templates/so/so-squid-template.json.jinja new file mode 100644 index 000000000..616607f52 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-squid-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-tomcat-template.json.jinja b/salt/elasticsearch/templates/so/so-tomcat-template.json.jinja new file mode 100644 index 000000000..797e71bcf --- /dev/null +++ b/salt/elasticsearch/templates/so/so-tomcat-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-tomcat:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-tomcat:refresh', '30s') %} +{ + "index_patterns": ["so-tomcat-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-zscaler-template.json.jinja b/salt/elasticsearch/templates/so/so-zscaler-template.json.jinja new file mode 100644 index 000000000..01d1cab2d --- /dev/null +++ b/salt/elasticsearch/templates/so/so-zscaler-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zscaler:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zscaler:refresh', '30s') %} +{ + "index_patterns": ["so-zscaler-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} From a27263435abd126f2f053dfc6bc237627e73d389 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 27 Aug 2021 14:41:04 -0400 Subject: [PATCH 015/136] Add Templates for all filebeat modules --- salt/elasticsearch/templates/so/so-snyk-template.json.jinja | 6 +++--- .../templates/so/so-sophos-template.json.jinja | 6 +++--- .../elasticsearch/templates/so/so-squid-template.json.jinja | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/elasticsearch/templates/so/so-snyk-template.json.jinja b/salt/elasticsearch/templates/so/so-snyk-template.json.jinja index 616607f52..0c2d291cc 100644 --- a/salt/elasticsearch/templates/so/so-snyk-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-snyk-template.json.jinja @@ -1,8 +1,8 @@ -{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-snyk:shards', 1) %} {%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} -{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-snyk:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-snyk-*"], "version":50001, "order":11, "settings":{ diff --git a/salt/elasticsearch/templates/so/so-sophos-template.json.jinja b/salt/elasticsearch/templates/so/so-sophos-template.json.jinja index 616607f52..689e19999 100644 --- a/salt/elasticsearch/templates/so/so-sophos-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-sophos-template.json.jinja @@ -1,8 +1,8 @@ -{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-sophos:shards', 1) %} {%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} -{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-sophos:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-sophos-*"], "version":50001, "order":11, "settings":{ diff --git a/salt/elasticsearch/templates/so/so-squid-template.json.jinja b/salt/elasticsearch/templates/so/so-squid-template.json.jinja index 616607f52..9398b8a99 100644 --- a/salt/elasticsearch/templates/so/so-squid-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-squid-template.json.jinja @@ -1,8 +1,8 @@ -{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-squid:shards', 1) %} {%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} -{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-squid:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-squid-*"], "version":50001, "order":11, "settings":{ From 72859adb13b049d8e6317ae386f3fef4fb194e11 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 27 Aug 2021 15:23:01 -0400 Subject: [PATCH 016/136] Fix typo in so-checkin --- salt/common/tools/sbin/so-checkin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-checkin b/salt/common/tools/sbin/so-checkin index c70701b71..4f0583906 100755 --- a/salt/common/tools/sbin/so-checkin +++ b/salt/common/tools/sbin/so-checkin @@ -17,4 +17,4 @@ . /usr/sbin/so-common -salt-call state.highstate -linfo +salt-call state.highstate -l info From 9605eda55959ca74e09b8c695211251caec016c7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 10:49:39 -0400 Subject: [PATCH 017/136] Close Curators for all modules --- salt/curator/files/action/so-aws-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-azure-close.yml | 29 +++++++++++++++++++ .../files/action/so-barracuda-close.yml | 29 +++++++++++++++++++ .../files/action/so-bluecoat-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-cef-close.yml | 29 +++++++++++++++++++ .../files/action/so-checkpoint-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-cisco-close.yml | 29 +++++++++++++++++++ .../files/action/so-cyberark-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-cylance-close.yml | 29 +++++++++++++++++++ .../files/action/so-elasticsearch-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-f5-close.yml | 29 +++++++++++++++++++ .../files/action/so-fortinet-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-gcp-close.yml | 29 +++++++++++++++++++ .../action/so-google_workspace-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-imperva-close.yml | 29 +++++++++++++++++++ .../files/action/so-infoblox-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-juniper-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-kibana-close.yml | 29 +++++++++++++++++++ .../files/action/so-logstash-close.yml | 29 +++++++++++++++++++ .../files/action/so-microsoft-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-misp-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-netflow-close.yml | 29 +++++++++++++++++++ .../files/action/so-netscout-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-o365-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-okta-close.yml | 29 +++++++++++++++++++ .../files/action/so-proofpoint-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-radware-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-redis-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-snort-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-snyk-close.yml | 29 +++++++++++++++++++ .../files/action/so-sonicwall-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-sophos-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-tomcat-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-zscaler-close.yml | 29 +++++++++++++++++++ 34 files changed, 986 insertions(+) create mode 100644 salt/curator/files/action/so-aws-close.yml create mode 100644 salt/curator/files/action/so-azure-close.yml create mode 100644 salt/curator/files/action/so-barracuda-close.yml create mode 100644 salt/curator/files/action/so-bluecoat-close.yml create mode 100644 salt/curator/files/action/so-cef-close.yml create mode 100644 salt/curator/files/action/so-checkpoint-close.yml create mode 100644 salt/curator/files/action/so-cisco-close.yml create mode 100644 salt/curator/files/action/so-cyberark-close.yml create mode 100644 salt/curator/files/action/so-cylance-close.yml create mode 100644 salt/curator/files/action/so-elasticsearch-close.yml create mode 100644 salt/curator/files/action/so-f5-close.yml create mode 100644 salt/curator/files/action/so-fortinet-close.yml create mode 100644 salt/curator/files/action/so-gcp-close.yml create mode 100644 salt/curator/files/action/so-google_workspace-close.yml create mode 100644 salt/curator/files/action/so-imperva-close.yml create mode 100644 salt/curator/files/action/so-infoblox-close.yml create mode 100644 salt/curator/files/action/so-juniper-close.yml create mode 100644 salt/curator/files/action/so-kibana-close.yml create mode 100644 salt/curator/files/action/so-logstash-close.yml create mode 100644 salt/curator/files/action/so-microsoft-close.yml create mode 100644 salt/curator/files/action/so-misp-close.yml create mode 100644 salt/curator/files/action/so-netflow-close.yml create mode 100644 salt/curator/files/action/so-netscout-close.yml create mode 100644 salt/curator/files/action/so-o365-close.yml create mode 100644 salt/curator/files/action/so-okta-close.yml create mode 100644 salt/curator/files/action/so-proofpoint-close.yml create mode 100644 salt/curator/files/action/so-radware-close.yml create mode 100644 salt/curator/files/action/so-redis-close.yml create mode 100644 salt/curator/files/action/so-snort-close.yml create mode 100644 salt/curator/files/action/so-snyk-close.yml create mode 100644 salt/curator/files/action/so-sonicwall-close.yml create mode 100644 salt/curator/files/action/so-sophos-close.yml create mode 100644 salt/curator/files/action/so-tomcat-close.yml create mode 100644 salt/curator/files/action/so-zscaler-close.yml diff --git a/salt/curator/files/action/so-aws-close.yml b/salt/curator/files/action/so-aws-close.yml new file mode 100644 index 000000000..44f0bfa97 --- /dev/null +++ b/salt/curator/files/action/so-aws-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-aws:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close aws indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-aws.*|so-aws.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-azure-close.yml b/salt/curator/files/action/so-azure-close.yml new file mode 100644 index 000000000..901b2c0ba --- /dev/null +++ b/salt/curator/files/action/so-azure-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-azure:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close azure indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-azure.*|so-azure.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-barracuda-close.yml b/salt/curator/files/action/so-barracuda-close.yml new file mode 100644 index 000000000..496832db7 --- /dev/null +++ b/salt/curator/files/action/so-barracuda-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-barracuda:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close barracuda indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-barracuda.*|so-barracuda.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-bluecoat-close.yml b/salt/curator/files/action/so-bluecoat-close.yml new file mode 100644 index 000000000..86d9277eb --- /dev/null +++ b/salt/curator/files/action/so-bluecoat-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-bluecoat:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close bluecoat indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-bluecoat.*|so-bluecoat.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-cef-close.yml b/salt/curator/files/action/so-cef-close.yml new file mode 100644 index 000000000..49e07f764 --- /dev/null +++ b/salt/curator/files/action/so-cef-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-cef:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close cef indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cef.*|so-cef.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-checkpoint-close.yml b/salt/curator/files/action/so-checkpoint-close.yml new file mode 100644 index 000000000..cffdf6473 --- /dev/null +++ b/salt/curator/files/action/so-checkpoint-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-checkpoint:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close checkpoint indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-checkpoint.*|so-checkpoint.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-cisco-close.yml b/salt/curator/files/action/so-cisco-close.yml new file mode 100644 index 000000000..cd1faade1 --- /dev/null +++ b/salt/curator/files/action/so-cisco-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-cisco:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close cisco indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cisco.*|so-cisco.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-cyberark-close.yml b/salt/curator/files/action/so-cyberark-close.yml new file mode 100644 index 000000000..e352e8355 --- /dev/null +++ b/salt/curator/files/action/so-cyberark-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-cyberark:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close cyberark indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cyberark.*|so-cyberark.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-cylance-close.yml b/salt/curator/files/action/so-cylance-close.yml new file mode 100644 index 000000000..d808569fb --- /dev/null +++ b/salt/curator/files/action/so-cylance-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-cylance:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close cylance indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cylance.*|so-cylance.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-elasticsearch-close.yml b/salt/curator/files/action/so-elasticsearch-close.yml new file mode 100644 index 000000000..3c4ff0dac --- /dev/null +++ b/salt/curator/files/action/so-elasticsearch-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-elasticsearch:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close elasticsearch indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-elasticsearch.*|so-elasticsearch.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-f5-close.yml b/salt/curator/files/action/so-f5-close.yml new file mode 100644 index 000000000..e1cdb48a1 --- /dev/null +++ b/salt/curator/files/action/so-f5-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-f5:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close f5 indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-f5.*|so-f5.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-fortinet-close.yml b/salt/curator/files/action/so-fortinet-close.yml new file mode 100644 index 000000000..e11fb86c6 --- /dev/null +++ b/salt/curator/files/action/so-fortinet-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-fortinet:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close fortinet indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-fortinet.*|so-fortinet.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-gcp-close.yml b/salt/curator/files/action/so-gcp-close.yml new file mode 100644 index 000000000..f9dd0af24 --- /dev/null +++ b/salt/curator/files/action/so-gcp-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-gcp:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close gcp indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-gcp.*|so-gcp.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-google_workspace-close.yml b/salt/curator/files/action/so-google_workspace-close.yml new file mode 100644 index 000000000..1ecda5893 --- /dev/null +++ b/salt/curator/files/action/so-google_workspace-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-google_workspace:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close google_workspace indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-google_workspace.*|so-google_workspace.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-imperva-close.yml b/salt/curator/files/action/so-imperva-close.yml new file mode 100644 index 000000000..55ec2e472 --- /dev/null +++ b/salt/curator/files/action/so-imperva-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-imperva:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close imperva indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-imperva.*|so-imperva.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-infoblox-close.yml b/salt/curator/files/action/so-infoblox-close.yml new file mode 100644 index 000000000..9fd4c5070 --- /dev/null +++ b/salt/curator/files/action/so-infoblox-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-infoblox:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close infoblox indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-infoblox.*|so-infoblox.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-juniper-close.yml b/salt/curator/files/action/so-juniper-close.yml new file mode 100644 index 000000000..466a51eca --- /dev/null +++ b/salt/curator/files/action/so-juniper-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-juniper:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close juniper indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-juniper.*|so-juniper.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-kibana-close.yml b/salt/curator/files/action/so-kibana-close.yml new file mode 100644 index 000000000..05583d853 --- /dev/null +++ b/salt/curator/files/action/so-kibana-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-logstash-close.yml b/salt/curator/files/action/so-logstash-close.yml new file mode 100644 index 000000000..23787e237 --- /dev/null +++ b/salt/curator/files/action/so-logstash-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-logstash:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close logstash indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-logstash.*|so-logstash.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-microsoft-close.yml b/salt/curator/files/action/so-microsoft-close.yml new file mode 100644 index 000000000..f4eaf738f --- /dev/null +++ b/salt/curator/files/action/so-microsoft-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-microsoft:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close microsoft indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-microsoft.*|so-microsoft.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-misp-close.yml b/salt/curator/files/action/so-misp-close.yml new file mode 100644 index 000000000..05583d853 --- /dev/null +++ b/salt/curator/files/action/so-misp-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-netflow-close.yml b/salt/curator/files/action/so-netflow-close.yml new file mode 100644 index 000000000..cc9ade28d --- /dev/null +++ b/salt/curator/files/action/so-netflow-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-netflow:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close netflow indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-netflow.*|so-netflow.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-netscout-close.yml b/salt/curator/files/action/so-netscout-close.yml new file mode 100644 index 000000000..05583d853 --- /dev/null +++ b/salt/curator/files/action/so-netscout-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-o365-close.yml b/salt/curator/files/action/so-o365-close.yml new file mode 100644 index 000000000..4dece060f --- /dev/null +++ b/salt/curator/files/action/so-o365-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-o365:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close o365 indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-o365.*|so-o365.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-okta-close.yml b/salt/curator/files/action/so-okta-close.yml new file mode 100644 index 000000000..10f7e4b60 --- /dev/null +++ b/salt/curator/files/action/so-okta-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-okta:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close okta indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-okta.*|so-okta.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-proofpoint-close.yml b/salt/curator/files/action/so-proofpoint-close.yml new file mode 100644 index 000000000..888c9fc64 --- /dev/null +++ b/salt/curator/files/action/so-proofpoint-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-proofpoint:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close proofpoint indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-proofpoint.*|so-proofpoint.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-radware-close.yml b/salt/curator/files/action/so-radware-close.yml new file mode 100644 index 000000000..05583d853 --- /dev/null +++ b/salt/curator/files/action/so-radware-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-redis-close.yml b/salt/curator/files/action/so-redis-close.yml new file mode 100644 index 000000000..b69935f21 --- /dev/null +++ b/salt/curator/files/action/so-redis-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-redis:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close redis indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-redis.*|so-redis.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-snort-close.yml b/salt/curator/files/action/so-snort-close.yml new file mode 100644 index 000000000..8f6209255 --- /dev/null +++ b/salt/curator/files/action/so-snort-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-snort:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close snort indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-snort.*|so-snort.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-snyk-close.yml b/salt/curator/files/action/so-snyk-close.yml new file mode 100644 index 000000000..05583d853 --- /dev/null +++ b/salt/curator/files/action/so-snyk-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-sonicwall-close.yml b/salt/curator/files/action/so-sonicwall-close.yml new file mode 100644 index 000000000..9cc23d3af --- /dev/null +++ b/salt/curator/files/action/so-sonicwall-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-sonicwall:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close sonicwall indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-sonicwall.*|so-sonicwall.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-sophos-close.yml b/salt/curator/files/action/so-sophos-close.yml new file mode 100644 index 000000000..b7574b996 --- /dev/null +++ b/salt/curator/files/action/so-sophos-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-sophos:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close sophos indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-sophos.*|so-sophos.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-tomcat-close.yml b/salt/curator/files/action/so-tomcat-close.yml new file mode 100644 index 000000000..ea0d95b0a --- /dev/null +++ b/salt/curator/files/action/so-tomcat-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-tomcat:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close tomcat indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-tomcat.*|so-tomcat.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-zscaler-close.yml b/salt/curator/files/action/so-zscaler-close.yml new file mode 100644 index 000000000..5a008a27d --- /dev/null +++ b/salt/curator/files/action/so-zscaler-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-zscaler:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close zscaler indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-zscaler.*|so-zscaler.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: From 3425a0fe78571b493495017d1df32a93e56edc19 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 11:12:21 -0400 Subject: [PATCH 018/136] Delete Curators for all modules --- salt/curator/files/action/so-aws-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-azure-delete.yml | 29 +++++++++++++++++++ .../files/action/so-barracuda-delete.yml | 29 +++++++++++++++++++ .../files/action/so-bluecoat-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-cef-delete.yml | 29 +++++++++++++++++++ .../files/action/so-checkpoint-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-cisco-delete.yml | 29 +++++++++++++++++++ .../files/action/so-cyberark-delete.yml | 29 +++++++++++++++++++ .../files/action/so-cylance-delete.yml | 29 +++++++++++++++++++ .../files/action/so-elasticsearch-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-f5-delete.yml | 29 +++++++++++++++++++ .../files/action/so-fortinet-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-gcp-delete.yml | 29 +++++++++++++++++++ .../files/action/so-google_workspace.yml | 29 +++++++++++++++++++ .../files/action/so-imperva-delete.yml | 29 +++++++++++++++++++ .../files/action/so-infoblox-delete.yml | 29 +++++++++++++++++++ .../files/action/so-juniper-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-kibana-delete.yml | 29 +++++++++++++++++++ .../files/action/so-logstash-delete.yml | 29 +++++++++++++++++++ .../files/action/so-microsoft-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-misp-close.yml | 6 ++-- salt/curator/files/action/so-misp-delete.yml | 29 +++++++++++++++++++ .../files/action/so-netflow-delete.yml | 29 +++++++++++++++++++ .../files/action/so-netscout-close.yml | 6 ++-- salt/curator/files/action/so-o365-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-okta.delete.yml | 29 +++++++++++++++++++ .../files/action/so-proofpoint-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-radware-close.yml | 6 ++-- .../files/action/so-radware-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-redis-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-snort-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-snyk-close.yml | 6 ++-- salt/curator/files/action/so-snyk-delete.yml | 29 +++++++++++++++++++ .../files/action/so-sonicwall-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-sophos-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-tomcat-delete.yml | 29 +++++++++++++++++++ .../files/action/so-zscaler-delete.yml | 29 +++++++++++++++++++ 37 files changed, 969 insertions(+), 12 deletions(-) create mode 100644 salt/curator/files/action/so-aws-delete.yml create mode 100644 salt/curator/files/action/so-azure-delete.yml create mode 100644 salt/curator/files/action/so-barracuda-delete.yml create mode 100644 salt/curator/files/action/so-bluecoat-delete.yml create mode 100644 salt/curator/files/action/so-cef-delete.yml create mode 100644 salt/curator/files/action/so-checkpoint-delete.yml create mode 100644 salt/curator/files/action/so-cisco-delete.yml create mode 100644 salt/curator/files/action/so-cyberark-delete.yml create mode 100644 salt/curator/files/action/so-cylance-delete.yml create mode 100644 salt/curator/files/action/so-elasticsearch-delete.yml create mode 100644 salt/curator/files/action/so-f5-delete.yml create mode 100644 salt/curator/files/action/so-fortinet-delete.yml create mode 100644 salt/curator/files/action/so-gcp-delete.yml create mode 100644 salt/curator/files/action/so-google_workspace.yml create mode 100644 salt/curator/files/action/so-imperva-delete.yml create mode 100644 salt/curator/files/action/so-infoblox-delete.yml create mode 100644 salt/curator/files/action/so-juniper-delete.yml create mode 100644 salt/curator/files/action/so-kibana-delete.yml create mode 100644 salt/curator/files/action/so-logstash-delete.yml create mode 100644 salt/curator/files/action/so-microsoft-delete.yml create mode 100644 salt/curator/files/action/so-misp-delete.yml create mode 100644 salt/curator/files/action/so-netflow-delete.yml create mode 100644 salt/curator/files/action/so-o365-delete.yml create mode 100644 salt/curator/files/action/so-okta.delete.yml create mode 100644 salt/curator/files/action/so-proofpoint-delete.yml create mode 100644 salt/curator/files/action/so-radware-delete.yml create mode 100644 salt/curator/files/action/so-redis-delete.yml create mode 100644 salt/curator/files/action/so-snort-delete.yml create mode 100644 salt/curator/files/action/so-snyk-delete.yml create mode 100644 salt/curator/files/action/so-sonicwall-delete.yml create mode 100644 salt/curator/files/action/so-sophos-delete.yml create mode 100644 salt/curator/files/action/so-tomcat-delete.yml create mode 100644 salt/curator/files/action/so-zscaler-delete.yml diff --git a/salt/curator/files/action/so-aws-delete.yml b/salt/curator/files/action/so-aws-delete.yml new file mode 100644 index 000000000..a67ee88b8 --- /dev/null +++ b/salt/curator/files/action/so-aws-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete aws indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-aws.*|so-aws.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-azure-delete.yml b/salt/curator/files/action/so-azure-delete.yml new file mode 100644 index 000000000..102a69d3d --- /dev/null +++ b/salt/curator/files/action/so-azure-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-azure:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete azure indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-azure.*|so-azure.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-barracuda-delete.yml b/salt/curator/files/action/so-barracuda-delete.yml new file mode 100644 index 000000000..49d472618 --- /dev/null +++ b/salt/curator/files/action/so-barracuda-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-barracuda:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete barracuda indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-barracuda.*|so-barracuda.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-bluecoat-delete.yml b/salt/curator/files/action/so-bluecoat-delete.yml new file mode 100644 index 000000000..318624416 --- /dev/null +++ b/salt/curator/files/action/so-bluecoat-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-bluecoat:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete bluecoat indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-bluecoat.*|so-bluecoat.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-cef-delete.yml b/salt/curator/files/action/so-cef-delete.yml new file mode 100644 index 000000000..0ee7d6501 --- /dev/null +++ b/salt/curator/files/action/so-cef-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cef:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete cef indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cef.*|so-cef.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-checkpoint-delete.yml b/salt/curator/files/action/so-checkpoint-delete.yml new file mode 100644 index 000000000..d1ac13efe --- /dev/null +++ b/salt/curator/files/action/so-checkpoint-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-checkpoint:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete checkpoint indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-checkpoint.*|so-checkpoint.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-cisco-delete.yml b/salt/curator/files/action/so-cisco-delete.yml new file mode 100644 index 000000000..bb5e06f7f --- /dev/null +++ b/salt/curator/files/action/so-cisco-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cisco:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete cisco indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cisco.*|so-cisco.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-cyberark-delete.yml b/salt/curator/files/action/so-cyberark-delete.yml new file mode 100644 index 000000000..784f6881e --- /dev/null +++ b/salt/curator/files/action/so-cyberark-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cyberark:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete cyberark indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cyberark.*|so-cyberark.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-cylance-delete.yml b/salt/curator/files/action/so-cylance-delete.yml new file mode 100644 index 000000000..54cf3938b --- /dev/null +++ b/salt/curator/files/action/so-cylance-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cylance:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete cylance indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cylance.*|so-cylance.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-elasticsearch-delete.yml b/salt/curator/files/action/so-elasticsearch-delete.yml new file mode 100644 index 000000000..05cc68abe --- /dev/null +++ b/salt/curator/files/action/so-elasticsearch-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-elasticsearch:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete elasticsearch indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-elasticsearch.*|so-elasticsearch.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-f5-delete.yml b/salt/curator/files/action/so-f5-delete.yml new file mode 100644 index 000000000..06704010a --- /dev/null +++ b/salt/curator/files/action/so-f5-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-f5:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete f5 indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-f5.*|so-f5.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-fortinet-delete.yml b/salt/curator/files/action/so-fortinet-delete.yml new file mode 100644 index 000000000..9379e47c2 --- /dev/null +++ b/salt/curator/files/action/so-fortinet-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-fortinet:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete fortinet indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-fortinet.*|so-fortinet.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-gcp-delete.yml b/salt/curator/files/action/so-gcp-delete.yml new file mode 100644 index 000000000..5c8ab33d8 --- /dev/null +++ b/salt/curator/files/action/so-gcp-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-gcp:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete gcp indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-gcp.*|so-gcp.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-google_workspace.yml b/salt/curator/files/action/so-google_workspace.yml new file mode 100644 index 000000000..923feda8f --- /dev/null +++ b/salt/curator/files/action/so-google_workspace.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-google_workspace:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete google_workspace indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-google_workspace.*|so-google_workspace.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-imperva-delete.yml b/salt/curator/files/action/so-imperva-delete.yml new file mode 100644 index 000000000..b5526e2fb --- /dev/null +++ b/salt/curator/files/action/so-imperva-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-imperva:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete imperva indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-imperva.*|so-imperva.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-infoblox-delete.yml b/salt/curator/files/action/so-infoblox-delete.yml new file mode 100644 index 000000000..0a7fdafbe --- /dev/null +++ b/salt/curator/files/action/so-infoblox-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-infoblox:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete infoblox indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-infoblox.*|so-infoblox.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-juniper-delete.yml b/salt/curator/files/action/so-juniper-delete.yml new file mode 100644 index 000000000..18abc86ac --- /dev/null +++ b/salt/curator/files/action/so-juniper-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-juniper:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete juniper indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-juniper.*|so-juniper.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-kibana-delete.yml b/salt/curator/files/action/so-kibana-delete.yml new file mode 100644 index 000000000..5a775b8de --- /dev/null +++ b/salt/curator/files/action/so-kibana-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-kibana:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete kibana indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-kibana.*|so-kibana.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-logstash-delete.yml b/salt/curator/files/action/so-logstash-delete.yml new file mode 100644 index 000000000..d9ff848da --- /dev/null +++ b/salt/curator/files/action/so-logstash-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-logstash:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete logstash indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-logstash.*|so-logstash.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-microsoft-delete.yml b/salt/curator/files/action/so-microsoft-delete.yml new file mode 100644 index 000000000..f1a854c83 --- /dev/null +++ b/salt/curator/files/action/so-microsoft-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-microsoft:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete microsoft indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-microsoft.*|so-microsoft.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-misp-close.yml b/salt/curator/files/action/so-misp-close.yml index 05583d853..e39781353 100644 --- a/salt/curator/files/action/so-misp-close.yml +++ b/salt/curator/files/action/so-misp-close.yml @@ -1,4 +1,4 @@ -{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-misp:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -10,7 +10,7 @@ actions: 1: action: close description: >- - Close IDS indices older than {{cur_close_days}} days. + Close misp indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -19,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-ids.*|so-ids.*)$' + value: '^(logstash-misp.*|so-misp.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-misp-delete.yml b/salt/curator/files/action/so-misp-delete.yml new file mode 100644 index 000000000..ceaa9c73d --- /dev/null +++ b/salt/curator/files/action/so-misp-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-misp:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete misp indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-misp.*|so-misp.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-netflow-delete.yml b/salt/curator/files/action/so-netflow-delete.yml new file mode 100644 index 000000000..5bc76ad15 --- /dev/null +++ b/salt/curator/files/action/so-netflow-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-netflow:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete netflow indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-netflow.*|so-netflow.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-netscout-close.yml b/salt/curator/files/action/so-netscout-close.yml index 05583d853..d99374d2f 100644 --- a/salt/curator/files/action/so-netscout-close.yml +++ b/salt/curator/files/action/so-netscout-close.yml @@ -1,4 +1,4 @@ -{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-netscout:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -10,7 +10,7 @@ actions: 1: action: close description: >- - Close IDS indices older than {{cur_close_days}} days. + Close netscout indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -19,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-ids.*|so-ids.*)$' + value: '^(logstash-netscout.*|so-netscout.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-o365-delete.yml b/salt/curator/files/action/so-o365-delete.yml new file mode 100644 index 000000000..13c7c1344 --- /dev/null +++ b/salt/curator/files/action/so-o365-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-o365:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete o365 indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-o365.*|so-o365.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-okta.delete.yml b/salt/curator/files/action/so-okta.delete.yml new file mode 100644 index 000000000..1beeb0fc0 --- /dev/null +++ b/salt/curator/files/action/so-okta.delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-okta:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete okta indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-okta.*|so-okta.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-proofpoint-delete.yml b/salt/curator/files/action/so-proofpoint-delete.yml new file mode 100644 index 000000000..903dde204 --- /dev/null +++ b/salt/curator/files/action/so-proofpoint-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-proofpoint:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete proofpoint indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-proofpoint.*|so-proofpoint.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-radware-close.yml b/salt/curator/files/action/so-radware-close.yml index 05583d853..59a7bbafd 100644 --- a/salt/curator/files/action/so-radware-close.yml +++ b/salt/curator/files/action/so-radware-close.yml @@ -1,4 +1,4 @@ -{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-radware:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -10,7 +10,7 @@ actions: 1: action: close description: >- - Close IDS indices older than {{cur_close_days}} days. + Close radware indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -19,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-ids.*|so-ids.*)$' + value: '^(logstash-radware.*|so-radware.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-radware-delete.yml b/salt/curator/files/action/so-radware-delete.yml new file mode 100644 index 000000000..1fe09cded --- /dev/null +++ b/salt/curator/files/action/so-radware-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-radware:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete radware indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-radware.*|so-radware.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-redis-delete.yml b/salt/curator/files/action/so-redis-delete.yml new file mode 100644 index 000000000..f6e73dce8 --- /dev/null +++ b/salt/curator/files/action/so-redis-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-redis:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete redis indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-redis.*|so-redis.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-snort-delete.yml b/salt/curator/files/action/so-snort-delete.yml new file mode 100644 index 000000000..50f68988b --- /dev/null +++ b/salt/curator/files/action/so-snort-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-snort:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete snort indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-snort.*|so-snort.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-snyk-close.yml b/salt/curator/files/action/so-snyk-close.yml index 05583d853..e13d8f98d 100644 --- a/salt/curator/files/action/so-snyk-close.yml +++ b/salt/curator/files/action/so-snyk-close.yml @@ -1,4 +1,4 @@ -{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-snyk:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -10,7 +10,7 @@ actions: 1: action: close description: >- - Close IDS indices older than {{cur_close_days}} days. + Close snyk indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -19,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-ids.*|so-ids.*)$' + value: '^(logstash-snyk.*|so-snyk.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-snyk-delete.yml b/salt/curator/files/action/so-snyk-delete.yml new file mode 100644 index 000000000..cec0b942f --- /dev/null +++ b/salt/curator/files/action/so-snyk-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-snyk:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete snyk indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-snyk.*|so-snyk.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-sonicwall-delete.yml b/salt/curator/files/action/so-sonicwall-delete.yml new file mode 100644 index 000000000..c7d38361f --- /dev/null +++ b/salt/curator/files/action/so-sonicwall-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-sonicwall:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete sonicwall indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-sonicwall.*|so-sonicwall.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-sophos-delete.yml b/salt/curator/files/action/so-sophos-delete.yml new file mode 100644 index 000000000..433df908a --- /dev/null +++ b/salt/curator/files/action/so-sophos-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-sophos:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete sophos indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-sophos.*|so-sophos.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-tomcat-delete.yml b/salt/curator/files/action/so-tomcat-delete.yml new file mode 100644 index 000000000..77035613f --- /dev/null +++ b/salt/curator/files/action/so-tomcat-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-tomcat:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete tomcat indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-tomcat.*|so-tomcat.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-zscaler-delete.yml b/salt/curator/files/action/so-zscaler-delete.yml new file mode 100644 index 000000000..238fea083 --- /dev/null +++ b/salt/curator/files/action/so-zscaler-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-zscaler:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete zscaler indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-zscaler.*|so-zscaler.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file From c09cebbd6b9f46334f70b633166e99620f2f19f8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 13:42:11 -0400 Subject: [PATCH 019/136] Add Actions for close and delete in cluster mode --- .../files/bin/so-curator-cluster-close | 44 +++++++++++++++++++ .../files/bin/so-curator-cluster-delete | 44 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 salt/curator/files/bin/so-curator-cluster-close create mode 100644 salt/curator/files/bin/so-curator-cluster-delete diff --git a/salt/curator/files/bin/so-curator-cluster-close b/salt/curator/files/bin/so-curator-cluster-close new file mode 100644 index 000000000..1a4124b53 --- /dev/null +++ b/salt/curator/files/bin/so-curator-cluster-close @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +APP=close +lf=/tmp/$APP-pidLockFile +# create empty lock file if none exists +cat /dev/null >> $lf +read lastPID < $lf +# if lastPID is not null and a process with that pid exists , exit +[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit +echo $$ > $lf + +{% from 'filebeat/map.jinja' import THIRDPARTY with context %} +{% from 'filebeat/map.jinja' import SO with context %} + +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; +{% for INDEX in THIRDPARTY -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1; +{% done -%} +{% for INDEX in SO -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} +{% done -%} \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-cluster-delete b/salt/curator/files/bin/so-curator-cluster-delete new file mode 100644 index 000000000..cccb270b0 --- /dev/null +++ b/salt/curator/files/bin/so-curator-cluster-delete @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +APP=delete +lf=/tmp/$APP-pidLockFile +# create empty lock file if none exists +cat /dev/null >> $lf +read lastPID < $lf +# if lastPID is not null and a process with that pid exists , exit +[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit +echo $$ > $lf + +{% from 'filebeat/map.jinja' import THIRDPARTY with context %} +{% from 'filebeat/map.jinja' import SO with context %} + +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-delete.yml > /dev/null 2>&1; +{% for INDEX in THIRDPARTY -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1; +{% done -%} +{% for INDEX in SO -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} +{% done -%} \ No newline at end of file From 476ecccbc10c8b0a414e2c1d060821eeccd6257d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:08:10 -0400 Subject: [PATCH 020/136] Add Actions for warm indices --- salt/curator/files/action/so-aws-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-azure-warm.yml | 24 +++++++++++++++ .../files/action/so-barracuda-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-beats-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-bluecoat-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-cef-warm.yml | 24 +++++++++++++++ .../files/action/so-checkpoint-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-cisco-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-cyberark-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-cylance-warm.yml | 24 +++++++++++++++ .../files/action/so-elasticsearch-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-f5-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-firewall-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-fortinet-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-gcp-warm.yml | 24 +++++++++++++++ ...ace.yml => so-google_workspace-delete.yml} | 0 .../files/action/so-google_workspace-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-ids-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-imperva-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-import-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-infoblox-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-juniper-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-kibana-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-logstash-warm.yml | 24 +++++++++++++++ .../files/action/so-microsoft-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-misp-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-netflow-warm.yml | 24 +++++++++++++++ .../files/action/so-netscout-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-netscout-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-okta-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-osquery-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-ossec-warm.yml | 24 +++++++++++++++ .../files/action/so-proofpoint-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-radware-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-redis-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-snort-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-snyk-warm.yml | 24 +++++++++++++++ .../files/action/so-sonicwall-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-sophos-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-strelka-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-syslog-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-tomcat-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-zeek-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-zscaler-warm.yml | 24 +++++++++++++++ salt/curator/files/action/warm.yml | 23 --------------- salt/curator/files/bin/so-cluster-warm | 27 +++++++++++++++++ 46 files changed, 1064 insertions(+), 23 deletions(-) create mode 100644 salt/curator/files/action/so-aws-warm.yml create mode 100644 salt/curator/files/action/so-azure-warm.yml create mode 100644 salt/curator/files/action/so-barracuda-warm.yml create mode 100644 salt/curator/files/action/so-beats-warm.yml create mode 100644 salt/curator/files/action/so-bluecoat-warm.yml create mode 100644 salt/curator/files/action/so-cef-warm.yml create mode 100644 salt/curator/files/action/so-checkpoint-warm.yml create mode 100644 salt/curator/files/action/so-cisco-warm.yml create mode 100644 salt/curator/files/action/so-cyberark-warm.yml create mode 100644 salt/curator/files/action/so-cylance-warm.yml create mode 100644 salt/curator/files/action/so-elasticsearch-warm.yml create mode 100644 salt/curator/files/action/so-f5-warm.yml create mode 100644 salt/curator/files/action/so-firewall-warm.yml create mode 100644 salt/curator/files/action/so-fortinet-warm.yml create mode 100644 salt/curator/files/action/so-gcp-warm.yml rename salt/curator/files/action/{so-google_workspace.yml => so-google_workspace-delete.yml} (100%) create mode 100644 salt/curator/files/action/so-google_workspace-warm.yml create mode 100644 salt/curator/files/action/so-ids-warm.yml create mode 100644 salt/curator/files/action/so-imperva-warm.yml create mode 100644 salt/curator/files/action/so-import-warm.yml create mode 100644 salt/curator/files/action/so-infoblox-warm.yml create mode 100644 salt/curator/files/action/so-juniper-warm.yml create mode 100644 salt/curator/files/action/so-kibana-warm.yml create mode 100644 salt/curator/files/action/so-logstash-warm.yml create mode 100644 salt/curator/files/action/so-microsoft-warm.yml create mode 100644 salt/curator/files/action/so-misp-warm.yml create mode 100644 salt/curator/files/action/so-netflow-warm.yml create mode 100644 salt/curator/files/action/so-netscout-delete.yml create mode 100644 salt/curator/files/action/so-netscout-warm.yml create mode 100644 salt/curator/files/action/so-okta-warm.yml create mode 100644 salt/curator/files/action/so-osquery-warm.yml create mode 100644 salt/curator/files/action/so-ossec-warm.yml create mode 100644 salt/curator/files/action/so-proofpoint-warm.yml create mode 100644 salt/curator/files/action/so-radware-warm.yml create mode 100644 salt/curator/files/action/so-redis-warm.yml create mode 100644 salt/curator/files/action/so-snort-warm.yml create mode 100644 salt/curator/files/action/so-snyk-warm.yml create mode 100644 salt/curator/files/action/so-sonicwall-warm.yml create mode 100644 salt/curator/files/action/so-sophos-warm.yml create mode 100644 salt/curator/files/action/so-strelka-warm.yml create mode 100644 salt/curator/files/action/so-syslog-warm.yml create mode 100644 salt/curator/files/action/so-tomcat-warm.yml create mode 100644 salt/curator/files/action/so-zeek-warm.yml create mode 100644 salt/curator/files/action/so-zscaler-warm.yml delete mode 100644 salt/curator/files/action/warm.yml create mode 100644 salt/curator/files/bin/so-cluster-warm diff --git a/salt/curator/files/action/so-aws-warm.yml b/salt/curator/files/action/so-aws-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-aws-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-azure-warm.yml b/salt/curator/files/action/so-azure-warm.yml new file mode 100644 index 000000000..d6f606125 --- /dev/null +++ b/salt/curator/files/action/so-azure-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-azure:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-azure + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-barracuda-warm.yml b/salt/curator/files/action/so-barracuda-warm.yml new file mode 100644 index 000000000..334a4114a --- /dev/null +++ b/salt/curator/files/action/so-barracuda-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-barracuda:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-barracuda + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-beats-warm.yml b/salt/curator/files/action/so-beats-warm.yml new file mode 100644 index 000000000..da9f76656 --- /dev/null +++ b/salt/curator/files/action/so-beats-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-beats:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-beats + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-bluecoat-warm.yml b/salt/curator/files/action/so-bluecoat-warm.yml new file mode 100644 index 000000000..47a8d712f --- /dev/null +++ b/salt/curator/files/action/so-bluecoat-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-bluecoat:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-bluecoat + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-cef-warm.yml b/salt/curator/files/action/so-cef-warm.yml new file mode 100644 index 000000000..0a79fd2ba --- /dev/null +++ b/salt/curator/files/action/so-cef-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cef:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-cef + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-checkpoint-warm.yml b/salt/curator/files/action/so-checkpoint-warm.yml new file mode 100644 index 000000000..0aaec1e19 --- /dev/null +++ b/salt/curator/files/action/so-checkpoint-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-checkpoint:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-checkpoint + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-cisco-warm.yml b/salt/curator/files/action/so-cisco-warm.yml new file mode 100644 index 000000000..a143a95c2 --- /dev/null +++ b/salt/curator/files/action/so-cisco-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cisco:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-cisco + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-cyberark-warm.yml b/salt/curator/files/action/so-cyberark-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-cyberark-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-cylance-warm.yml b/salt/curator/files/action/so-cylance-warm.yml new file mode 100644 index 000000000..c9da7e68a --- /dev/null +++ b/salt/curator/files/action/so-cylance-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cylance:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-cylance + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-elasticsearch-warm.yml b/salt/curator/files/action/so-elasticsearch-warm.yml new file mode 100644 index 000000000..9d82fc27b --- /dev/null +++ b/salt/curator/files/action/so-elasticsearch-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-elasticsearch:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-elasticsearch + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-f5-warm.yml b/salt/curator/files/action/so-f5-warm.yml new file mode 100644 index 000000000..12fbbe7ad --- /dev/null +++ b/salt/curator/files/action/so-f5-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-f5:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-f5 + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-firewall-warm.yml b/salt/curator/files/action/so-firewall-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-firewall-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-fortinet-warm.yml b/salt/curator/files/action/so-fortinet-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-fortinet-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-gcp-warm.yml b/salt/curator/files/action/so-gcp-warm.yml new file mode 100644 index 000000000..3bb9eee80 --- /dev/null +++ b/salt/curator/files/action/so-gcp-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-gcp:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-gcp + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-google_workspace.yml b/salt/curator/files/action/so-google_workspace-delete.yml similarity index 100% rename from salt/curator/files/action/so-google_workspace.yml rename to salt/curator/files/action/so-google_workspace-delete.yml diff --git a/salt/curator/files/action/so-google_workspace-warm.yml b/salt/curator/files/action/so-google_workspace-warm.yml new file mode 100644 index 000000000..7eb2d883f --- /dev/null +++ b/salt/curator/files/action/so-google_workspace-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-google_workspace:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-google_workspace + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-ids-warm.yml b/salt/curator/files/action/so-ids-warm.yml new file mode 100644 index 000000000..0edad5b5b --- /dev/null +++ b/salt/curator/files/action/so-ids-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-ids:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-ids + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-imperva-warm.yml b/salt/curator/files/action/so-imperva-warm.yml new file mode 100644 index 000000000..0297d5cd6 --- /dev/null +++ b/salt/curator/files/action/so-imperva-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-imperva:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-imperva + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-import-warm.yml b/salt/curator/files/action/so-import-warm.yml new file mode 100644 index 000000000..3a6fa3d3d --- /dev/null +++ b/salt/curator/files/action/so-import-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-import:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-import + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-infoblox-warm.yml b/salt/curator/files/action/so-infoblox-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-infoblox-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-juniper-warm.yml b/salt/curator/files/action/so-juniper-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-juniper-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-kibana-warm.yml b/salt/curator/files/action/so-kibana-warm.yml new file mode 100644 index 000000000..b5674c8c3 --- /dev/null +++ b/salt/curator/files/action/so-kibana-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-kibana:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-kibana + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-logstash-warm.yml b/salt/curator/files/action/so-logstash-warm.yml new file mode 100644 index 000000000..826bf2975 --- /dev/null +++ b/salt/curator/files/action/so-logstash-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-logstash:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-logstash + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-microsoft-warm.yml b/salt/curator/files/action/so-microsoft-warm.yml new file mode 100644 index 000000000..551d0cb56 --- /dev/null +++ b/salt/curator/files/action/so-microsoft-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-microsoft:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-microsoft + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-misp-warm.yml b/salt/curator/files/action/so-misp-warm.yml new file mode 100644 index 000000000..af29975b0 --- /dev/null +++ b/salt/curator/files/action/so-misp-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-misp:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-misp + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-netflow-warm.yml b/salt/curator/files/action/so-netflow-warm.yml new file mode 100644 index 000000000..ea57bb72b --- /dev/null +++ b/salt/curator/files/action/so-netflow-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-netflow:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-netflow + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-netscout-delete.yml b/salt/curator/files/action/so-netscout-delete.yml new file mode 100644 index 000000000..3c0e249b5 --- /dev/null +++ b/salt/curator/files/action/so-netscout-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-netscout:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete netscout indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-netscout.*|so-netscout.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-netscout-warm.yml b/salt/curator/files/action/so-netscout-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-netscout-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-okta-warm.yml b/salt/curator/files/action/so-okta-warm.yml new file mode 100644 index 000000000..75764860d --- /dev/null +++ b/salt/curator/files/action/so-okta-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-okta:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-okta + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-osquery-warm.yml b/salt/curator/files/action/so-osquery-warm.yml new file mode 100644 index 000000000..156a83c7a --- /dev/null +++ b/salt/curator/files/action/so-osquery-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-osquery:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-osquery + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-ossec-warm.yml b/salt/curator/files/action/so-ossec-warm.yml new file mode 100644 index 000000000..769d6cbea --- /dev/null +++ b/salt/curator/files/action/so-ossec-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-ossec:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-ossec + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-proofpoint-warm.yml b/salt/curator/files/action/so-proofpoint-warm.yml new file mode 100644 index 000000000..8304ae41a --- /dev/null +++ b/salt/curator/files/action/so-proofpoint-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-proofpoint:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-proofpoint + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-radware-warm.yml b/salt/curator/files/action/so-radware-warm.yml new file mode 100644 index 000000000..8d4337aaf --- /dev/null +++ b/salt/curator/files/action/so-radware-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-radware:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-radware + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-redis-warm.yml b/salt/curator/files/action/so-redis-warm.yml new file mode 100644 index 000000000..a5b1055c3 --- /dev/null +++ b/salt/curator/files/action/so-redis-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-redis:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-redis + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-snort-warm.yml b/salt/curator/files/action/so-snort-warm.yml new file mode 100644 index 000000000..3bbc977e2 --- /dev/null +++ b/salt/curator/files/action/so-snort-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-snort:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-snort + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-snyk-warm.yml b/salt/curator/files/action/so-snyk-warm.yml new file mode 100644 index 000000000..f9b10bbdd --- /dev/null +++ b/salt/curator/files/action/so-snyk-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-snyk:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-snyk + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-sonicwall-warm.yml b/salt/curator/files/action/so-sonicwall-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-sonicwall-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-sophos-warm.yml b/salt/curator/files/action/so-sophos-warm.yml new file mode 100644 index 000000000..40cc60084 --- /dev/null +++ b/salt/curator/files/action/so-sophos-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-sophos:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-sophos + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-strelka-warm.yml b/salt/curator/files/action/so-strelka-warm.yml new file mode 100644 index 000000000..cfa88b0c1 --- /dev/null +++ b/salt/curator/files/action/so-strelka-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-strelka:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-strelka + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-syslog-warm.yml b/salt/curator/files/action/so-syslog-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-syslog-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-tomcat-warm.yml b/salt/curator/files/action/so-tomcat-warm.yml new file mode 100644 index 000000000..8fb7884c0 --- /dev/null +++ b/salt/curator/files/action/so-tomcat-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-tomcat:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-tomcat + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-zeek-warm.yml b/salt/curator/files/action/so-zeek-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-zeek-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-zscaler-warm.yml b/salt/curator/files/action/so-zscaler-warm.yml new file mode 100644 index 000000000..8a7d8187a --- /dev/null +++ b/salt/curator/files/action/so-zscaler-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-zscaler:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-zscaler + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/warm.yml b/salt/curator/files/action/warm.yml deleted file mode 100644 index efa2196b2..000000000 --- a/salt/curator/files/action/warm.yml +++ /dev/null @@ -1,23 +0,0 @@ -#actions: -# 1: -# action: allocation -# description: "Apply shard allocation filtering rules to the specified indices" -# options: -# key: box_type -# value: warm -# allocation_type: require -# wait_for_completion: true -# timeout_override: -# continue_if_exception: false -# disable_action: false -# filters: -# - filtertype: pattern -# kind: prefix -# value: so- -# - filtertype: age -# source: name -# direction: older -# timestring: '%Y.%m.%d' -# unit: days -# unit_count: 3 - diff --git a/salt/curator/files/bin/so-cluster-warm b/salt/curator/files/bin/so-cluster-warm new file mode 100644 index 000000000..c8234c7c9 --- /dev/null +++ b/salt/curator/files/bin/so-cluster-warm @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +APP=warm +lf=/tmp/$APP-pidLockFile +# create empty lock file if none exists +cat /dev/null >> $lf +read lastPID < $lf +# if lastPID is not null and a process with that pid exists , exit +[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit +echo $$ > $lf + +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/warm.yml > /dev/null 2>&1 \ No newline at end of file From ed116cf850bcbfaffcfd4b1aa04e96828781c5ad Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:09:26 -0400 Subject: [PATCH 021/136] Add Actions for warm indices --- salt/curator/files/action/so-cyberark-warm.yml | 4 ++-- salt/curator/files/action/so-firewall-warm.yml | 4 ++-- salt/curator/files/action/so-fortinet-warm.yml | 4 ++-- salt/curator/files/action/so-netscout-warm.yml | 4 ++-- salt/curator/files/action/so-sonicwall-warm.yml | 4 ++-- salt/curator/files/action/so-syslog-warm.yml | 4 ++-- salt/curator/files/action/so-zeek-warm.yml | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/salt/curator/files/action/so-cyberark-warm.yml b/salt/curator/files/action/so-cyberark-warm.yml index 5369ed9a9..8eae0b542 100644 --- a/salt/curator/files/action/so-cyberark-warm.yml +++ b/salt/curator/files/action/so-cyberark-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cyberark:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-cyberark - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-firewall-warm.yml b/salt/curator/files/action/so-firewall-warm.yml index 5369ed9a9..2e9643dc3 100644 --- a/salt/curator/files/action/so-firewall-warm.yml +++ b/salt/curator/files/action/so-firewall-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-firewall:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-firewall - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-fortinet-warm.yml b/salt/curator/files/action/so-fortinet-warm.yml index 5369ed9a9..db9a6f2db 100644 --- a/salt/curator/files/action/so-fortinet-warm.yml +++ b/salt/curator/files/action/so-fortinet-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-fortinet:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-fortinet - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-netscout-warm.yml b/salt/curator/files/action/so-netscout-warm.yml index 5369ed9a9..1b93c3118 100644 --- a/salt/curator/files/action/so-netscout-warm.yml +++ b/salt/curator/files/action/so-netscout-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-netscout:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-netscout - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-sonicwall-warm.yml b/salt/curator/files/action/so-sonicwall-warm.yml index 5369ed9a9..fa8ceb3e4 100644 --- a/salt/curator/files/action/so-sonicwall-warm.yml +++ b/salt/curator/files/action/so-sonicwall-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-sonicwall:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-sonicwall - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-syslog-warm.yml b/salt/curator/files/action/so-syslog-warm.yml index 5369ed9a9..e5ebb2fa6 100644 --- a/salt/curator/files/action/so-syslog-warm.yml +++ b/salt/curator/files/action/so-syslog-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-syslog:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-syslog - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-zeek-warm.yml b/salt/curator/files/action/so-zeek-warm.yml index 5369ed9a9..2b4b6a729 100644 --- a/salt/curator/files/action/so-zeek-warm.yml +++ b/salt/curator/files/action/so-zeek-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-zeek - filtertype: age source: name direction: older From f6f4375e1396956124d875388e6176a8fb2873da Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:34:26 -0400 Subject: [PATCH 022/136] Add logic for cronjobs --- ...o-cluster-warm => so-curator-cluster-warm} | 0 salt/curator/init.sls | 127 ++++++++++++------ 2 files changed, 85 insertions(+), 42 deletions(-) rename salt/curator/files/bin/{so-cluster-warm => so-curator-cluster-warm} (100%) diff --git a/salt/curator/files/bin/so-cluster-warm b/salt/curator/files/bin/so-curator-cluster-warm similarity index 100% rename from salt/curator/files/bin/so-cluster-warm rename to salt/curator/files/bin/so-curator-cluster-warm diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 1acea5d25..57da8bfa1 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -5,6 +5,9 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set REMOVECURATORCRON = False %} +{% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} +{% set HOTWARM = salt['pillar.get']('elasticsearch:hot_warm_enabled', False) %} + {% if grains['role'] in ['so-eval', 'so-node', 'so-managersearch', 'so-heavynode', 'so-standalone', 'so-manager'] %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} {% from "curator/map.jinja" import CURATOROPTIONS with context %} @@ -148,62 +151,102 @@ delete_so-curator_so-status: {% if REMOVECURATORCRON %} so-curatorcloseddeletecron: - cron.absent: - - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 - - user: root + cron.absent: + - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 + - user: root so-curatorclosecron: - cron.absent: - - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 - - user: root + cron.absent: + - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 + - user: root so-curatordeletecron: - cron.absent: - - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 - - user: root + cron.absent: + - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 + - user: root {% else %} + {% if TRUECLUSTER is sameas true %} so-curatorcloseddeletecron: - cron.present: - - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 - - user: root - - minute: '*' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' + cron.absent: + - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 + - user: root so-curatorclosecron: - cron.present: - - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 - - user: root - - minute: '*' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' + cron.absent: + - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 + - user: root so-curatordeletecron: - cron.present: - - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 - - user: root - - minute: '*' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' + cron.absent: + - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 + - user: root + +so-curatorclusterclose: + cron.present: + - name: /usr/sbin/so-curator-cluster-close > /opt/so/log/curator/cron-close.log 2>&1 + - user: root + - minute: '*' + - hour: '*/1' + - daymonth: '*' + - month: '*' + - dayweek: '*' +so-curatorclusterdelete: + cron.present: + - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-close.log 2>&1 + - user: root + - minute: '*' + - hour: '*/1' + - daymonth: '*' + - month: '*' + - dayweek: '*' + {% if HOTWARM is sameas true %} +so-curatorclusterwarm: + cron.present: + - name: /usr/sbin/so-curator-cluster-warm > /opt/so/log/curator/cron-close.log 2>&1 + - user: root + - minute: '*' + - hour: '*/1' + - daymonth: '*' + - month: '*' + - dayweek: '*' + {% endif %%} + + {% else %} +so-curatorcloseddeletecron: + cron.present: + - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + +so-curatorclosecron: + cron.present: + - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + +so-curatordeletecron: + cron.present: + - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + + {% endif %} {% endif %} - -# Begin Curator Cron Jobs - -# Close -# Delete -# Hot Warm -# Segment Merge - -# End Curator Cron Jobs {% endif %} {% else %} From 9f51f02ab48538a014b866012991da26ed953e4f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:40:09 -0400 Subject: [PATCH 023/136] Add logic for cronjobs --- salt/curator/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 57da8bfa1..8900c35d8 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -187,7 +187,7 @@ so-curatorclusterclose: cron.present: - name: /usr/sbin/so-curator-cluster-close > /opt/so/log/curator/cron-close.log 2>&1 - user: root - - minute: '*' + - minute: '2' - hour: '*/1' - daymonth: '*' - month: '*' @@ -197,7 +197,7 @@ so-curatorclusterdelete: cron.present: - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-close.log 2>&1 - user: root - - minute: '*' + - minute: '2' - hour: '*/1' - daymonth: '*' - month: '*' @@ -207,7 +207,7 @@ so-curatorclusterwarm: cron.present: - name: /usr/sbin/so-curator-cluster-warm > /opt/so/log/curator/cron-close.log 2>&1 - user: root - - minute: '*' + - minute: '2' - hour: '*/1' - daymonth: '*' - month: '*' From dc25ed2594b01430ca23722b6bcd9deb1c01043a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:43:48 -0400 Subject: [PATCH 024/136] Add logic for cronjobs --- salt/curator/init.sls | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 8900c35d8..b7b580af7 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -167,22 +167,7 @@ so-curatordeletecron: {% else %} - {% if TRUECLUSTER is sameas true %} -so-curatorcloseddeletecron: - cron.absent: - - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 - - user: root - -so-curatorclosecron: - cron.absent: - - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 - - user: root - -so-curatordeletecron: - cron.absent: - - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 - - user: root - + {% if TRUECLUSTER is sameas true %} so-curatorclusterclose: cron.present: - name: /usr/sbin/so-curator-cluster-close > /opt/so/log/curator/cron-close.log 2>&1 From c12f380bc348c10b4b1adda9d65f22964f29539f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:51:34 -0400 Subject: [PATCH 025/136] Only enable curator on Manager in true cluster --- salt/curator/map.jinja | 4 ++-- salt/top.sls | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/curator/map.jinja b/salt/curator/map.jinja index 470143a4f..bbcff2a9c 100644 --- a/salt/curator/map.jinja +++ b/salt/curator/map.jinja @@ -4,10 +4,10 @@ {% do CURATOROPTIONS.update({'manage_sostatus': True}) %} # don't start the docker container if curator is disabled via pillar -{% if not ENABLED or grains.id.split('_')|last == 'manager'%} +{% if not ENABLED or (TRUECLUSTER and grains.id.split('_')|last == 'searchnode') or (not TRUECLUSTER and grains.id.split('_')|last == 'manager') %} {% do CURATOROPTIONS.update({'start': False}) %} {% do CURATOROPTIONS.update({'status': 'absent'}) %} - {% if grains.id.split('_')|last == 'manager' %} + {% if (TRUECLUSTER and grains.id.split('_')|last == 'searchnode') or (not TRUECLUSTER and grains.id.split('_')|last == 'manager') %} {% do CURATOROPTIONS.update({'manage_sostatus': False}) %} {% endif %} {% else %} diff --git a/salt/top.sls b/salt/top.sls index b38d315d8..e2c980e1d 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -189,6 +189,7 @@ base: {%- if KIBANA %} - kibana {%- endif %} + - curator {%- if ELASTALERT %} - elastalert {%- endif %} From 5b9ef3bc0d2ed5b8881398ebe89a45b7019e9fd2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:55:44 -0400 Subject: [PATCH 026/136] Only enable curator on Manager in true cluster --- salt/curator/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index b7b580af7..10cffe680 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -197,7 +197,7 @@ so-curatorclusterwarm: - daymonth: '*' - month: '*' - dayweek: '*' - {% endif %%} + {% endif %} {% else %} so-curatorcloseddeletecron: From 0c8e11dc9f3903b8cd51e26642d217aa15bd7eb9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:13:05 -0400 Subject: [PATCH 027/136] Only enable curator on Manager in true cluster --- salt/curator/init.sls | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 10cffe680..95e62ac95 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -92,6 +92,30 @@ curdel: - group: 939 - mode: 755 +curclusterclose: + file.managed: + - name: /usr/sbin/so-curator-cluster-close + - source: salt://curator/files/bin/so-curator-cluster-close + - user: 934 + - group: 939 + - mode: 755 + +curclusterdelete: + file.managed: + - name: /usr/sbin/so-curator-cluster-delete + - source: salt://curator/files/bin/so-curator-cluster-delete + - user: 934 + - group: 939 + - mode: 755 + +curclustercwarm: + file.managed: + - name: /usr/sbin/so-curator-cluster-warm + - source: salt://curator/files/bin/so-curator-cluster-warm + - user: 934 + - group: 939 + - mode: 755 + so-curator: docker_container.{{ CURATOROPTIONS.status }}: {% if CURATOROPTIONS.status == 'running' %} From 958575c22a2fdbe4f09910c65960305fa1561b73 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:17:55 -0400 Subject: [PATCH 028/136] Only enable curator on Manager in true cluster --- salt/curator/init.sls | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 95e62ac95..54fcff3a6 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -99,6 +99,7 @@ curclusterclose: - user: 934 - group: 939 - mode: 755 + - template: jinja curclusterdelete: file.managed: @@ -107,6 +108,7 @@ curclusterdelete: - user: 934 - group: 939 - mode: 755 + - template: jinja curclustercwarm: file.managed: @@ -115,7 +117,8 @@ curclustercwarm: - user: 934 - group: 939 - mode: 755 - + - template: jinja + so-curator: docker_container.{{ CURATOROPTIONS.status }}: {% if CURATOROPTIONS.status == 'running' %} From 62edfd0b7fde61491cf75c5ec499e1ca5ada16ec Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:20:42 -0400 Subject: [PATCH 029/136] Only enable curator on Manager in true cluster --- salt/curator/files/bin/so-curator-cluster-close | 4 ++-- salt/curator/files/bin/so-curator-cluster-delete | 4 ++-- salt/curator/files/bin/so-curator-cluster-warm | 16 +++++++++++++++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/salt/curator/files/bin/so-curator-cluster-close b/salt/curator/files/bin/so-curator-cluster-close index 1a4124b53..b2043d3b6 100644 --- a/salt/curator/files/bin/so-curator-cluster-close +++ b/salt/curator/files/bin/so-curator-cluster-close @@ -38,7 +38,7 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; {% for INDEX in THIRDPARTY -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1; -{% done -%} +{% endfor -%} {% for INDEX in SO -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} -{% done -%} \ No newline at end of file +{% endfor -%} \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-cluster-delete b/salt/curator/files/bin/so-curator-cluster-delete index cccb270b0..df7f2c4bf 100644 --- a/salt/curator/files/bin/so-curator-cluster-delete +++ b/salt/curator/files/bin/so-curator-cluster-delete @@ -38,7 +38,7 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-delete.yml > /dev/null 2>&1; {% for INDEX in THIRDPARTY -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1; -{% done -%} +{% endfor -%} {% for INDEX in SO -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} -{% done -%} \ No newline at end of file +{% endfor -%} \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-cluster-warm b/salt/curator/files/bin/so-curator-cluster-warm index c8234c7c9..db54b7fe9 100644 --- a/salt/curator/files/bin/so-curator-cluster-warm +++ b/salt/curator/files/bin/so-curator-cluster-warm @@ -24,4 +24,18 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/warm.yml > /dev/null 2>&1 \ No newline at end of file +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-warm.yml > /dev/null 2>&1; +{% for INDEX in THIRDPARTY -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1; +{% endfor -%} +{% for INDEX in SO -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} +{% endfor -%} \ No newline at end of file From e7b04b862fb2eb21b358fa6e20d682db6d0404aa Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:21:48 -0400 Subject: [PATCH 030/136] Only enable curator on Manager in true cluster --- salt/curator/files/bin/so-curator-cluster-warm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/curator/files/bin/so-curator-cluster-warm b/salt/curator/files/bin/so-curator-cluster-warm index db54b7fe9..ec08a31c0 100644 --- a/salt/curator/files/bin/so-curator-cluster-warm +++ b/salt/curator/files/bin/so-curator-cluster-warm @@ -24,6 +24,9 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf +{% from 'filebeat/map.jinja' import THIRDPARTY with context %} +{% from 'filebeat/map.jinja' import SO with context %} + docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-warm.yml > /dev/null 2>&1; From e09aa4e5d4b359cd7885f58df003429bae495912 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:35:19 -0400 Subject: [PATCH 031/136] Only enable curator on Manager in true cluster --- salt/curator/files/bin/so-curator-cluster-close | 4 ++-- salt/curator/files/bin/so-curator-cluster-delete | 4 ++-- salt/curator/files/bin/so-curator-cluster-warm | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/curator/files/bin/so-curator-cluster-close b/salt/curator/files/bin/so-curator-cluster-close index b2043d3b6..7b8c41c9e 100644 --- a/salt/curator/files/bin/so-curator-cluster-close +++ b/salt/curator/files/bin/so-curator-cluster-close @@ -36,9 +36,9 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; -{% for INDEX in THIRDPARTY -%} +{% for INDEX in THIRDPARTY.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1; {% endfor -%} -{% for INDEX in SO -%} +{% for INDEX in SO.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} {% endfor -%} \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-cluster-delete b/salt/curator/files/bin/so-curator-cluster-delete index df7f2c4bf..21195da5d 100644 --- a/salt/curator/files/bin/so-curator-cluster-delete +++ b/salt/curator/files/bin/so-curator-cluster-delete @@ -36,9 +36,9 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-delete.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-delete.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-delete.yml > /dev/null 2>&1; -{% for INDEX in THIRDPARTY -%} +{% for INDEX in THIRDPARTY.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1; {% endfor -%} -{% for INDEX in SO -%} +{% for INDEX in SO.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} {% endfor -%} \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-cluster-warm b/salt/curator/files/bin/so-curator-cluster-warm index ec08a31c0..4348ba2e3 100644 --- a/salt/curator/files/bin/so-curator-cluster-warm +++ b/salt/curator/files/bin/so-curator-cluster-warm @@ -36,9 +36,9 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-warm.yml > /dev/null 2>&1; -{% for INDEX in THIRDPARTY -%} +{% for INDEX in THIRDPARTY.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1; {% endfor -%} -{% for INDEX in SO -%} +{% for INDEX in SO.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} {% endfor -%} \ No newline at end of file From 61ebedc0e9439c43c57c4ae8f2fff118984f16e6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:56:08 -0400 Subject: [PATCH 032/136] Only enable curator on Manager in true cluster --- salt/curator/files/action/so-infoblox-warm.yml | 4 ++-- salt/curator/files/action/so-kibana-close.yml | 6 +++--- salt/curator/init.sls | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/curator/files/action/so-infoblox-warm.yml b/salt/curator/files/action/so-infoblox-warm.yml index 5369ed9a9..a2f571b7a 100644 --- a/salt/curator/files/action/so-infoblox-warm.yml +++ b/salt/curator/files/action/so-infoblox-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-infoblox:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-infoblox - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-kibana-close.yml b/salt/curator/files/action/so-kibana-close.yml index 05583d853..7347fb01c 100644 --- a/salt/curator/files/action/so-kibana-close.yml +++ b/salt/curator/files/action/so-kibana-close.yml @@ -1,4 +1,4 @@ -{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-kibana:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -10,7 +10,7 @@ actions: 1: action: close description: >- - Close IDS indices older than {{cur_close_days}} days. + Close kibana indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -19,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-ids.*|so-ids.*)$' + value: '^(logstash-kibana.*|so-kibana.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 54fcff3a6..87e1fec33 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -207,7 +207,7 @@ so-curatorclusterclose: so-curatorclusterdelete: cron.present: - - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-close.log 2>&1 + - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-delete.log 2>&1 - user: root - minute: '2' - hour: '*/1' @@ -217,7 +217,7 @@ so-curatorclusterdelete: {% if HOTWARM is sameas true %} so-curatorclusterwarm: cron.present: - - name: /usr/sbin/so-curator-cluster-warm > /opt/so/log/curator/cron-close.log 2>&1 + - name: /usr/sbin/so-curator-cluster-warm > /opt/so/log/curator/cron-warm.log 2>&1 - user: root - minute: '2' - hour: '*/1' From 527a793e9419ada93a38257e90cf51b97ea90a16 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:59:41 -0400 Subject: [PATCH 033/136] Only enable curator on Manager in true cluster --- salt/curator/files/action/so-o365-warm.yml | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 salt/curator/files/action/so-o365-warm.yml diff --git a/salt/curator/files/action/so-o365-warm.yml b/salt/curator/files/action/so-o365-warm.yml new file mode 100644 index 000000000..cbb7bc24e --- /dev/null +++ b/salt/curator/files/action/so-o365-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-o365:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-o365 + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + From 007b15979a11a2297bf0badec2fae50c2ac1e7c6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Sep 2021 09:25:14 -0400 Subject: [PATCH 034/136] Non Cluster honor closed indices values --- salt/curator/files/bin/so-curator-close | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/salt/curator/files/bin/so-curator-close b/salt/curator/files/bin/so-curator-close index 6f7b96790..c1e940b4d 100644 --- a/salt/curator/files/bin/so-curator-close +++ b/salt/curator/files/bin/so-curator-close @@ -23,5 +23,22 @@ read lastPID < $lf # if lastPID is not null and a process with that pid exists , exit [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf +{% from 'filebeat/map.jinja' import THIRDPARTY with context %} +{% from 'filebeat/map.jinja' import SO with context %} -/usr/sbin/so-curator-closed-delete > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1 +/usr/sbin/so-curator-closed-delete > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; +{% for INDEX in THIRDPARTY.modules.keys() -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1; +{% endfor -%} +{% for INDEX in SO.modules.keys() -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} +{% endfor -%} From 324e6b12e2bbad13686c172e09559542de810231 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Sep 2021 09:32:32 -0400 Subject: [PATCH 035/136] Add jinja template --- salt/curator/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 87e1fec33..c604571ba 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -83,6 +83,7 @@ curclose: - user: 934 - group: 939 - mode: 755 + - template: jinja curdel: file.managed: From 576c893eb3731817ecf9eb709467e6d6ab640f3a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 1 Sep 2021 15:08:53 -0400 Subject: [PATCH 036/136] Exit on missing file argument --- salt/common/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 60e0ce93f..87fe56784 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -788,6 +788,7 @@ main() { check_airgap if [[ $is_airgap -eq 0 && $UNATTENDED == true && -z $ISOLOC ]]; then echo "Missing file argument for unattended airgap upgrade." + exit 1 fi echo "Checking to see if this is a manager." echo "" From 446821e9fdd657d580787e39835b2d55a368aa58 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 1 Sep 2021 15:11:18 -0400 Subject: [PATCH 037/136] Use exit code 0 when printing error message before exiting soup --- salt/common/tools/sbin/soup | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 87fe56784..de26e73ea 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -127,7 +127,7 @@ EOF if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then echo "Something went wrong trying to mount the ISO." echo "Ensure you verify the ISO that you downloaded." - exit 1 + exit 0 else echo "ISO has been mounted!" fi @@ -140,14 +140,14 @@ EOF if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then echo "Something went wrong trying to mount the device." echo "Ensure you verify the ISO that you downloaded." - exit 1 + exit 0 else echo "Device has been mounted!" fi else echo "Could not find Security Onion ISO content at ${ISOLOC}" echo "Ensure the path you entered is correct, and that you verify the ISO that you downloaded." - exit 1 + exit 0 fi fi } @@ -157,7 +157,7 @@ airgap_update_dockers() { # Let's copy the tarball if [[ ! -f $AGDOCKER/registry.tar ]]; then echo "Unable to locate registry. Exiting" - exit 1 + exit 0 else echo "Stopping the registry docker" docker stop so-dockerregistry @@ -631,7 +631,7 @@ upgrade_space() { clean_dockers if ! verify_upgradespace; then echo "There is not enough space to perform the upgrade. Please free up space and try again" - exit 1 + exit 0 fi else echo "You have enough space for upgrade. Proceeding with soup." @@ -788,7 +788,7 @@ main() { check_airgap if [[ $is_airgap -eq 0 && $UNATTENDED == true && -z $ISOLOC ]]; then echo "Missing file argument for unattended airgap upgrade." - exit 1 + exit 0 fi echo "Checking to see if this is a manager." echo "" @@ -885,7 +885,7 @@ main() { echo "Once the issue is resolved, run soup again." echo "Exiting." echo "" - exit 1 + exit 0 else echo "Salt upgrade success." echo "" From 556bad6925f59cb98bba37f88d46bda080856b2d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Sep 2021 15:13:43 -0400 Subject: [PATCH 038/136] Add eventfields for new default logs --- salt/soc/files/soc/hunt.eventfields.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/hunt.eventfields.json b/salt/soc/files/soc/hunt.eventfields.json index e8af03a5c..354aa31ce 100644 --- a/salt/soc/files/soc/hunt.eventfields.json +++ b/salt/soc/files/soc/hunt.eventfields.json @@ -40,5 +40,9 @@ ":strelka:file": ["soc_timestamp", "file.name", "file.size", "hash.md5", "file.source", "file.mime_type", "log.id.fuid" ], ":suricata:": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "rule.name", "rule.category", "event.severity_label", "log.id.uid", "network.community_id" ], ":sysmon:": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "source.hostname", "event.dataset", "process.executable", "user.name" ], - ":windows_eventlog:": ["soc_timestamp", "user.name" ] + ":windows_eventlog:": ["soc_timestamp", "user.name" ], + ":elasticsearch:": ["soc_timestamp", "agent.name", "message", "log.level", "metadata.version", "metadata.pipeline", "event.dataset" ], + ":kibana:": ["soc_timestamp", "host.name", "message", "kibana.log.meta.req.headers.x-real-ip", "event.dataset" ], + "::rootcheck": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ], + "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ] } From 0ad3d826ebf7ea3cccc08c8a392a5e064b1bde1f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Sep 2021 07:07:36 -0400 Subject: [PATCH 039/136] Invalid password message should also mention that dollar signs are not allowed --- salt/common/tools/sbin/so-fleet-user-add | 2 +- salt/common/tools/sbin/so-fleet-user-update | 2 +- salt/common/tools/sbin/so-thehive-user-add | 2 +- salt/common/tools/sbin/so-thehive-user-update | 2 +- setup/so-whiptail | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-fleet-user-add b/salt/common/tools/sbin/so-fleet-user-add index e905424a9..9d80c2076 100755 --- a/salt/common/tools/sbin/so-fleet-user-add +++ b/salt/common/tools/sbin/so-fleet-user-add @@ -42,7 +42,7 @@ fi read -rs FLEET_PASS if ! check_password "$FLEET_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." + echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." exit 2 fi diff --git a/salt/common/tools/sbin/so-fleet-user-update b/salt/common/tools/sbin/so-fleet-user-update index e6a142d1d..793f7b622 100755 --- a/salt/common/tools/sbin/so-fleet-user-update +++ b/salt/common/tools/sbin/so-fleet-user-update @@ -52,7 +52,7 @@ fi read -rs FLEET_PASS if ! check_password "$FLEET_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." + echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." exit 2 fi diff --git a/salt/common/tools/sbin/so-thehive-user-add b/salt/common/tools/sbin/so-thehive-user-add index 5d174fdf1..e26dc58fc 100755 --- a/salt/common/tools/sbin/so-thehive-user-add +++ b/salt/common/tools/sbin/so-thehive-user-add @@ -42,7 +42,7 @@ fi read -rs THEHIVE_PASS if ! check_password "$THEHIVE_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." + echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." exit 2 fi diff --git a/salt/common/tools/sbin/so-thehive-user-update b/salt/common/tools/sbin/so-thehive-user-update index 6df199f6a..fdda5eaa7 100755 --- a/salt/common/tools/sbin/so-thehive-user-update +++ b/salt/common/tools/sbin/so-thehive-user-update @@ -42,7 +42,7 @@ fi read -rs THEHIVE_PASS if ! check_password "$THEHIVE_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." + echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." exit 2 fi diff --git a/setup/so-whiptail b/setup/so-whiptail index 780411841..125335729 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -843,7 +843,7 @@ whiptail_invalid_pass_characters_warning() { [ -n "$TESTING" ] && return - whiptail --title "$whiptail_title" --msgbox "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." 8 75 + whiptail --title "$whiptail_title" --msgbox "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." 8 75 } whiptail_invalid_pass_warning() { From 10126bb7ef05aeed6545dde9a9151faee9b9677e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Sep 2021 09:44:57 -0400 Subject: [PATCH 040/136] Auth enhancements --- .../tools/sbin/so-elasticsearch-roles-load | 57 ++++++++ salt/common/tools/sbin/so-user | 132 ++++++++++++++---- salt/elasticsearch/init.sls | 29 +++- salt/elasticsearch/roles/analyst.json | 45 ++++++ salt/soc/files/kratos/schema.json | 4 - salt/soc/files/soc/custom_roles | 20 +++ salt/soc/files/soc/soc.json | 8 ++ salt/soc/init.sls | 11 ++ setup/so-functions | 2 +- 9 files changed, 269 insertions(+), 39 deletions(-) create mode 100644 salt/common/tools/sbin/so-elasticsearch-roles-load create mode 100644 salt/elasticsearch/roles/analyst.json create mode 100644 salt/soc/files/soc/custom_roles diff --git a/salt/common/tools/sbin/so-elasticsearch-roles-load b/salt/common/tools/sbin/so-elasticsearch-roles-load new file mode 100644 index 000000000..3b0f580fe --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-roles-load @@ -0,0 +1,57 @@ +#!/bin/bash +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +{%- set mainint = salt['pillar.get']('host:mainint') %} +{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} + +default_conf_dir=/opt/so/conf +ELASTICSEARCH_HOST="{{ MYIP }}" +ELASTICSEARCH_PORT=9200 + +# Define a default directory to load roles from +ELASTICSEARCH_ROLES="$default_conf_dir/elasticsearch/roles/" + +# Wait for ElasticSearch to initialize +echo -n "Waiting for ElasticSearch..." +COUNT=0 +ELASTICSEARCH_CONNECTED="no" +while [[ "$COUNT" -le 240 ]]; do + {{ ELASTICCURL }} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi +done +if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo +fi + +cd ${ELASTICSEARCH_ROLES} + +echo "Loading templates..." +for role in *; do + name=$(echo "$role" | cut -d. -f1) + so-elasticsearch-query security/roles/$name -XPUT -d @"$role" +done + +cd - >/dev/null diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 742c3ca5d..7ec094efb 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -18,11 +18,17 @@ source $(dirname $0)/so-common +DEFAULT_ROLE=analyst + if [[ $# -lt 1 || $# -gt 2 ]]; then - echo "Usage: $0 [email]" + echo "Usage: $0 [email] [role]" + echo "" + echo " where is one of the following:" echo "" echo " list: Lists all user email addresses currently defined in the identity system" echo " add: Adds a new user to the identity system; requires 'email' parameter" + echo " addrole: Grants a role to an existing user; requires 'email' and 'role' parameters" + echo " delrole: Removes a role from an existing user; requires 'email' and 'role' parameters" echo " update: Updates a user's password; requires 'email' parameter" echo " enable: Enables a user; requires 'email' parameter" echo " disable: Disables a user; requires 'email' parameter" @@ -36,6 +42,7 @@ fi operation=$1 email=$2 +role=$3 kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434} databasePath=${KRATOS_DB_PATH:-/opt/so/conf/kratos/db/db.sqlite} @@ -138,10 +145,9 @@ function updatePassword() { function createElasticFile() { filename=$1 - tmpFile=${filename} - truncate -s 0 "$tmpFile" - chmod 600 "$tmpFile" - chown "${esUID}:${esGID}" "$tmpFile" + truncate -s 0 "$filename" + chmod 600 "$filename" + chown "${esUID}:${esGID}" "$filename" } function syncElasticSystemUser() { @@ -174,28 +180,15 @@ function syncElasticSystemRole() { function syncElastic() { echo "Syncing users between SOC and Elastic..." usersTmpFile="${elasticUsersFile}.tmp" - rolesTmpFile="${elasticRolesFile}.tmp" createElasticFile "${usersTmpFile}" - createElasticFile "${rolesTmpFile}" authPillarJson=$(lookup_salt_value "auth" "elasticsearch" "pillar" "json") syncElasticSystemUser "$authPillarJson" "so_elastic_user" "$usersTmpFile" - syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$rolesTmpFile" - syncElasticSystemUser "$authPillarJson" "so_kibana_user" "$usersTmpFile" - syncElasticSystemRole "$authPillarJson" "so_kibana_user" "superuser" "$rolesTmpFile" - syncElasticSystemUser "$authPillarJson" "so_logstash_user" "$usersTmpFile" - syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$rolesTmpFile" - syncElasticSystemUser "$authPillarJson" "so_beats_user" "$usersTmpFile" - syncElasticSystemRole "$authPillarJson" "so_beats_user" "superuser" "$rolesTmpFile" - syncElasticSystemUser "$authPillarJson" "so_monitor_user" "$usersTmpFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "monitoring_user" "$rolesTmpFile" if [[ -f "$databasePath" ]]; then # Generate the new users file @@ -207,23 +200,12 @@ function syncElastic() { jq -r '.user + ":" + .data.hashed_password' \ >> "$usersTmpFile" [[ $? != 0 ]] && fail "Unable to read credential hashes from database" - - # Generate the new users_roles file - - echo "select 'superuser:' || ici.identifier " \ - "from identity_credential_identifiers ici, identity_credentials ic " \ - "where ici.identity_credential_id=ic.id and instr(ic.config, 'hashed_password') " \ - "order by ici.identifier;" | \ - sqlite3 "$databasePath" \ - >> "$rolesTmpFile" - [[ $? != 0 ]] && fail "Unable to read credential IDs from database" else echo "Database file does not exist yet, skipping users export" fi if [[ -s "${usersTmpFile}" ]]; then mv "${usersTmpFile}" "${elasticUsersFile}" - mv "${rolesTmpFile}" "${elasticRolesFile}" if [[ -z "$SKIP_STATE_APPLY" ]]; then echo "Elastic state will be re-applied to affected minions. This may take several minutes..." @@ -252,11 +234,73 @@ function listUsers() { response=$(curl -Ss -L ${kratosUrl}/identities) [[ $? != 0 ]] && fail "Unable to communicate with Kratos" - echo "${response}" | jq -r ".[] | .verifiable_addresses[0].value" | sort + users=$(echo "${response}" | jq -r ".[] | .verifiable_addresses[0].value" | sort) + for user in $users; do + roles=$(grep "$user" users_roles | cut -d: -f1 | tr '\n' ' ') + echo "$user: $roles" + done +} + +function addUserRole() { + email=$1 + role=$2 + + return adjustUserRole "$email" "$role" "add" +} + +function deleteUserRole() { + email=$1 + role=$2 + + return adjustUserRole "$email" "$role" "del" +} + +function adjustUserRole() { + email=$1 + role=$2 + op=$3 + + identityId=$(findIdByEmail "$email") + [[ ${identityId} == "" ]] && fail "User not found" + + if [ ! -f "$filename" ]; then + rolesTmpFile="${elasticRolesFile}.tmp" + createElasticFile "${rolesTmpFile}" + authPillarJson=$(lookup_salt_value "auth" "elasticsearch" "pillar" "json") + syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_kibana_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_beats_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "monitoring_user" "$rolesTmpFile" + mv "${rolesTmpFile}" "${elasticRolesFile}" + fi + + filename="$elasticRolesFile" + grep "$role:" "$elasticRolesFile" | grep "$email" && hasRole=1 + if [[ "$op" == "add" ]]; then + if [[ "$hasRole" -eq 1 ]]; then + fail "User '$email' already has the role: $role" + else + echo "$role:$email" >> "$filename" + fi + elif [[ "$op" == "del" ]]; then + if [[ "$hasRole" -ne 1 ]]; then + fail "User '$email' does not have the role: $role" + else + sed -i "/^$role:$email\$/d" "$filename" + fi + else + echo "Unsupported role adjustment operation: $op" + exit 1 + fi + return 0 } function createUser() { email=$1 + role=$1 now=$(date -u +%FT%TZ) addUserJson=$(cat < /opt/so/conf/elasticsearch/users && chown 930:930 /opt/so/conf/elasticsearch/users && chmod 600 /opt/so/conf/elasticsearch/users + - name: cat /opt/so/conf/elasticsearch/users.tmp > /opt/so/conf/elasticsearch/users && chown 930:939 /opt/so/conf/elasticsearch/users && chmod 660 /opt/so/conf/elasticsearch/users - onchanges: - file: /opt/so/conf/elasticsearch/users.tmp @@ -201,7 +218,7 @@ auth_users_roles_inode: require: - file: auth_users_roles cmd.run: - - name: cat /opt/so/conf/elasticsearch/users_roles.tmp > /opt/so/conf/elasticsearch/users_roles && chown 930:930 /opt/so/conf/elasticsearch/users_roles && chmod 600 /opt/so/conf/elasticsearch/users_roles + - name: cat /opt/so/conf/elasticsearch/users_roles.tmp > /opt/so/conf/elasticsearch/users_roles && chown 930:939 /opt/so/conf/elasticsearch/users_roles && chmod 660 /opt/so/conf/elasticsearch/users_roles - onchanges: - file: /opt/so/conf/elasticsearch/users_roles.tmp @@ -283,7 +300,7 @@ so-elasticsearch-pipelines: - file: esyml - file: so-elasticsearch-pipelines-file -{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node', 'so-import'] and TEMPLATES %} +{% if TEMPLATES %} so-elasticsearch-templates: cmd.run: - name: /usr/sbin/so-elasticsearch-templates-load @@ -291,6 +308,12 @@ so-elasticsearch-templates: - template: jinja {% endif %} +so-elasticsearch-roles-load: + cmd.run: + - name: /usr/sbin/so-elasticsearch-roles-load + - cwd: /opt/so + - template: jinja + {% endif %} {# if grains['role'] != 'so-helix' #} {% else %} diff --git a/salt/elasticsearch/roles/analyst.json b/salt/elasticsearch/roles/analyst.json new file mode 100644 index 000000000..f81c5a67e --- /dev/null +++ b/salt/elasticsearch/roles/analyst.json @@ -0,0 +1,45 @@ +{ + "elasticsearch": { + "cluster": [ + "cancel_task", + "create_snapshot", + "monitor", + "monitor_data_frame_transforms", + "monitor_ml", + "monitor_rollup", + "monitor_snapshot", + "monitor_text_structure", + "monitor_transform", + "monitor_watcher", + "read_ccr", + "read_ilm", + "read_pipeline", + "read_slm" + ], + "indices": [ + { + "names": [ + "so-*" + ], + "privileges": [ + "read", + "read_cross_cluster", + "monitor", + "view_index_metadata" + ] + } + ], + "run_as": [] + }, + "kibana": [ + { + "spaces": [ + "*" + ], + "base": [ + "read" + ], + "feature": {} + } + ] +} \ No newline at end of file diff --git a/salt/soc/files/kratos/schema.json b/salt/soc/files/kratos/schema.json index 986086936..19ee2197c 100644 --- a/salt/soc/files/kratos/schema.json +++ b/salt/soc/files/kratos/schema.json @@ -31,10 +31,6 @@ "type": "string", "title": "Last Name" }, - "role": { - "type": "string", - "title": "Role" - }, "status": { "type": "string", "title": "Status" diff --git a/salt/soc/files/soc/custom_roles b/salt/soc/files/soc/custom_roles new file mode 100644 index 000000000..80ae7b147 --- /dev/null +++ b/salt/soc/files/soc/custom_roles @@ -0,0 +1,20 @@ +# Define custom business role mappings, or remove mappings that come with +# the default SOC deployment. +# +# IMPORTANT: This file should be copied from the salt/default tree into +# the salt/local tree (preserving the same directory structure). +# Failure to do this will result in the customizations being +# overwritten on future upgrades. +# +# Syntax => prebuiltRoleX: customRoleY: op +# Explanation => roleY and roleZ are adjusted permissions of roleX, op is: +# + add the new permissions/role mappings (default) +# - remove existing prebuilt permissions +# +# In the example below, we will define a new role for junior analysts, +# that is nearly identical to the analyst role that comes with SOC, with the +# exception that it removes their ability to obtain details about other +# analysts in the system. +# +# analyst: jr_analyst +# user-monitor: jr_analyst:- diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index fc6d5f28d..6119f0e6b 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -85,6 +85,14 @@ "statickeyauth": { "anonymousCidr": "{{ DNET }}/24", "apiKey": "{{ SENSORONIKEY }}" + }, + "staticrbac": { + "roleFiles": [ + "rbac/permissions", + "rbac/roles", + "rbac/users_roles", + "rbac/custom_roles" + ] } }, "client": { diff --git a/salt/soc/init.sls b/salt/soc/init.sls index b8cdb09ba..c3c466849 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -62,6 +62,15 @@ soccustom: - mode: 600 - template: jinja +soccustomroles: + file.managed: + - name: /opt/so/conf/soc/custom_roles + - source: salt://soc/files/soc/custom_roles + - user: 939 + - group: 939 + - mode: 600 + - template: jinja + # we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate' # is in the minion config. That line is added before the final highstate during setup sosyncusers: @@ -81,6 +90,8 @@ so-soc: - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro + - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro + - /opt/so/conf/elasticsearch/users_roles:/opt/sensoroni/rbac/users_roles:ro - /opt/so/log/soc/:/opt/sensoroni/logs/:rw {%- if salt['pillar.get']('nodestab', {}) %} - extra_hosts: diff --git a/setup/so-functions b/setup/so-functions index 9a64a561e..d851c80b1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -121,7 +121,7 @@ add_web_user() { { echo "Attempting to add administrator user for web interface..."; export SKIP_STATE_APPLY=true - echo "$WEBPASSWD1" | /usr/sbin/so-user add "$WEBUSER"; + echo "$WEBPASSWD1" | /usr/sbin/so-user add "$WEBUSER" "superuser"; unset SKIP_STATE_APPLY echo "Add user result: $?"; } >> "/root/so-user-add.log" 2>&1 From 56894b95813a4566ca18ab035882c513961ad5e4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 2 Sep 2021 11:15:32 -0400 Subject: [PATCH 041/136] Add logic to check unattended flag when checking if updates are available --- salt/common/tools/sbin/soup | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index de26e73ea..17339fdd5 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -289,25 +289,31 @@ check_os_updates() { OSUPDATES=$(yum -q list updates | wc -l) fi if [[ "$OSUPDATES" -gt 0 ]]; then - echo $NEEDUPDATES - echo "" - read -p "Press U to update OS packages (recommended), C to continue without updates, or E to exit: " confirm - - if [[ "$confirm" == [cC] ]]; then + if [[ -z $UNATTENDED ]]; then + echo "$NEEDUPDATES" + echo "" + read -rp "Press U to update OS packages (recommended), C to continue without updates, or E to exit: " confirm + if [[ "$confirm" == [cC] ]]; then echo "Continuing without updating packages" - elif [[ "$confirm" == [uU] ]]; then + elif [[ "$confirm" == [uU] ]]; then echo "Applying Grid Updates" - set +e - run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.' - set -e - else + update_flag=true + else echo "Exiting soup" exit 0 + fi + else + update_flag=true fi else - echo "Looks like you have an updated OS" + echo "Looks like you have an updated OS" + fi + + if [[ $update_flag == true ]]; then + set +e + run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.' + set -e fi - } clean_dockers() { @@ -787,7 +793,7 @@ main() { echo "" check_airgap if [[ $is_airgap -eq 0 && $UNATTENDED == true && -z $ISOLOC ]]; then - echo "Missing file argument for unattended airgap upgrade." + echo "Missing file argument (-f ) for unattended airgap upgrade." exit 0 fi echo "Checking to see if this is a manager." From 2bf471054bb3c6aa0afde1c430ac7933c21427ce Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 2 Sep 2021 11:46:18 -0400 Subject: [PATCH 042/136] Cloudtrail Event Fields --- salt/soc/files/soc/hunt.eventfields.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/hunt.eventfields.json b/salt/soc/files/soc/hunt.eventfields.json index 354aa31ce..fa5f41ded 100644 --- a/salt/soc/files/soc/hunt.eventfields.json +++ b/salt/soc/files/soc/hunt.eventfields.json @@ -44,5 +44,6 @@ ":elasticsearch:": ["soc_timestamp", "agent.name", "message", "log.level", "metadata.version", "metadata.pipeline", "event.dataset" ], ":kibana:": ["soc_timestamp", "host.name", "message", "kibana.log.meta.req.headers.x-real-ip", "event.dataset" ], "::rootcheck": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ], - "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ] + "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ], + ":aws:": ["soc_timestamp", "aws.cloudtrail.event_category", "aws.cloudtrail.event_type", "event.provider", "event.action", "event.outcome", "cloud.region", "user.name", "s ource.ip", "source.geo.region_iso_code" ] } From 1cee603ee4f56d4c3f1917f41122dbd79589a2f2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 2 Sep 2021 13:24:04 -0400 Subject: [PATCH 043/136] Squid event fields --- salt/soc/files/soc/hunt.eventfields.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/hunt.eventfields.json b/salt/soc/files/soc/hunt.eventfields.json index fa5f41ded..9eac130e8 100644 --- a/salt/soc/files/soc/hunt.eventfields.json +++ b/salt/soc/files/soc/hunt.eventfields.json @@ -45,5 +45,6 @@ ":kibana:": ["soc_timestamp", "host.name", "message", "kibana.log.meta.req.headers.x-real-ip", "event.dataset" ], "::rootcheck": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ], "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ], - ":aws:": ["soc_timestamp", "aws.cloudtrail.event_category", "aws.cloudtrail.event_type", "event.provider", "event.action", "event.outcome", "cloud.region", "user.name", "s ource.ip", "source.geo.region_iso_code" ] + ":aws:": ["soc_timestamp", "aws.cloudtrail.event_category", "aws.cloudtrail.event_type", "event.provider", "event.action", "event.outcome", "cloud.region", "user.name", "source.ip", "source.geo.region_iso_code" ], + ":squid:": ["soc_timestamp", "url.original", "destination.ip", "destination.geo.country_iso_code", "user.name", "source.ip" ] } From 9f5dafd560c47b00b6bc71e2cc6bd3078d59975f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 2 Sep 2021 13:48:18 -0400 Subject: [PATCH 044/136] More Event Fields --- salt/soc/files/soc/hunt.eventfields.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/hunt.eventfields.json b/salt/soc/files/soc/hunt.eventfields.json index 9eac130e8..e519e78b6 100644 --- a/salt/soc/files/soc/hunt.eventfields.json +++ b/salt/soc/files/soc/hunt.eventfields.json @@ -30,7 +30,7 @@ "::software": ["soc_timestamp", "source.ip", "software.name", "software.type" ], "::ssh": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ssh.version", "ssh.hassh_version", "ssh.direction", "ssh.client", "ssh.server", "log.id.uid" ], "::ssl": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ssl.server_name", "ssl.certificate.subject", "ssl.validation_status", "ssl.version", "log.id.uid" ], - "::syslog": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "syslog.facility", "network.protocol", "syslog.severity", "log.id.uid" ], + ":zeek:syslog": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "syslog.facility", "network.protocol", "syslog.severity", "log.id.uid" ], "::tunnels": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "tunnel_type", "action", "log.id.uid" ], "::weird": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "weird.name", "log.id.uid" ], "::x509": ["soc_timestamp", "x509.certificate.subject", "x509.certificate.key.type", "x509.certificate.key.length", "x509.certificate.issuer", "log.id.fuid" ], @@ -44,7 +44,9 @@ ":elasticsearch:": ["soc_timestamp", "agent.name", "message", "log.level", "metadata.version", "metadata.pipeline", "event.dataset" ], ":kibana:": ["soc_timestamp", "host.name", "message", "kibana.log.meta.req.headers.x-real-ip", "event.dataset" ], "::rootcheck": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ], + "::ossec": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ], "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ], + ":syslog:syslog": ["soc_timestamp", "host.name", "metadata.ip_address", "real_message", "syslog.priority", "syslog.application" ], ":aws:": ["soc_timestamp", "aws.cloudtrail.event_category", "aws.cloudtrail.event_type", "event.provider", "event.action", "event.outcome", "cloud.region", "user.name", "source.ip", "source.geo.region_iso_code" ], ":squid:": ["soc_timestamp", "url.original", "destination.ip", "destination.geo.country_iso_code", "user.name", "source.ip" ] } From c4d402d8b429a8da407b8557a2e567b45bdac1c1 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Sep 2021 15:45:47 -0400 Subject: [PATCH 045/136] Ensure role file exists before ES state is run --- salt/common/tools/sbin/so-user | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 7ec094efb..d60b04567 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -150,6 +150,23 @@ function createElasticFile() { chown "${esUID}:${esGID}" "$filename" } +function ensureRoleFileExists() { + if [ ! -f "$elasticRolesFile" ]; then + echo "Creating new roles file: $elasticRolesFile" + rolesTmpFile="${elasticRolesFile}.tmp" + createElasticFile "${rolesTmpFile}" + authPillarJson=$(lookup_salt_value "auth" "elasticsearch" "pillar" "json") + syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_kibana_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_beats_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "monitoring_user" "$rolesTmpFile" + mv "${rolesTmpFile}" "${elasticRolesFile}" + fi +} + function syncElasticSystemUser() { json=$1 userid=$2 @@ -179,6 +196,8 @@ function syncElasticSystemRole() { function syncElastic() { echo "Syncing users between SOC and Elastic..." + ensureRoleFileExists + usersTmpFile="${elasticUsersFile}.tmp" createElasticFile "${usersTmpFile}" @@ -263,19 +282,7 @@ function adjustUserRole() { identityId=$(findIdByEmail "$email") [[ ${identityId} == "" ]] && fail "User not found" - if [ ! -f "$filename" ]; then - rolesTmpFile="${elasticRolesFile}.tmp" - createElasticFile "${rolesTmpFile}" - authPillarJson=$(lookup_salt_value "auth" "elasticsearch" "pillar" "json") - syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_kibana_user" "superuser" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_beats_user" "superuser" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "monitoring_user" "$rolesTmpFile" - mv "${rolesTmpFile}" "${elasticRolesFile}" - fi + ensureRoleFileExists filename="$elasticRolesFile" grep "$role:" "$elasticRolesFile" | grep "$email" && hasRole=1 From ce70380f0ff4d19130709b96e7d7b4bd4edf39f8 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Sep 2021 17:59:33 -0400 Subject: [PATCH 046/136] resolve so-user errors from recent auth changes --- salt/common/tools/sbin/so-user | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index d60b04567..d7f9c2e6b 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -20,7 +20,7 @@ source $(dirname $0)/so-common DEFAULT_ROLE=analyst -if [[ $# -lt 1 || $# -gt 2 ]]; then +if [[ $# -lt 1 || $# -gt 3 ]]; then echo "Usage: $0 [email] [role]" echo "" echo " where is one of the following:" @@ -197,7 +197,7 @@ function syncElasticSystemRole() { function syncElastic() { echo "Syncing users between SOC and Elastic..." ensureRoleFileExists - + usersTmpFile="${elasticUsersFile}.tmp" createElasticFile "${usersTmpFile}" @@ -255,7 +255,7 @@ function listUsers() { users=$(echo "${response}" | jq -r ".[] | .verifiable_addresses[0].value" | sort) for user in $users; do - roles=$(grep "$user" users_roles | cut -d: -f1 | tr '\n' ' ') + roles=$(grep "$user" "$elasticRolesFile" | cut -d: -f1 | tr '\n' ' ') echo "$user: $roles" done } @@ -264,14 +264,14 @@ function addUserRole() { email=$1 role=$2 - return adjustUserRole "$email" "$role" "add" + adjustUserRole "$email" "$role" "add" } function deleteUserRole() { email=$1 role=$2 - return adjustUserRole "$email" "$role" "del" + adjustUserRole "$email" "$role" "del" } function adjustUserRole() { @@ -285,9 +285,10 @@ function adjustUserRole() { ensureRoleFileExists filename="$elasticRolesFile" - grep "$role:" "$elasticRolesFile" | grep "$email" && hasRole=1 + hasRole=0 + grep "$role:" "$elasticRolesFile" | grep -q "$email" && hasRole=1 if [[ "$op" == "add" ]]; then - if [[ "$hasRole" -eq 1 ]]; then + if [[ "$hasRole" == "1" ]]; then fail "User '$email' already has the role: $role" else echo "$role:$email" >> "$filename" @@ -299,15 +300,13 @@ function adjustUserRole() { sed -i "/^$role:$email\$/d" "$filename" fi else - echo "Unsupported role adjustment operation: $op" - exit 1 + fail "Unsupported role adjustment operation: $op" fi - return 0 } function createUser() { email=$1 - role=$1 + role=$2 now=$(date -u +%FT%TZ) addUserJson=$(cat < Date: Thu, 2 Sep 2021 19:12:32 -0400 Subject: [PATCH 047/136] Consolidate password validation messaging --- salt/common/tools/sbin/so-common | 9 +++++++++ salt/common/tools/sbin/so-fleet-user-add | 5 +---- salt/common/tools/sbin/so-thehive-user-add | 5 +---- salt/common/tools/sbin/so-user | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 7ad74ad49..fe97c9b27 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -99,6 +99,15 @@ check_password() { return $? } +check_password_and_exit() { + local password=$1 + if ! check_password "$password"; then + echo "Password is invalid. Do not include single quotes, double quotes, dollar signs, and backslashes in the password." + exit 2 + fi + return 0 +} + check_elastic_license() { [ -n "$TESTING" ] && return diff --git a/salt/common/tools/sbin/so-fleet-user-add b/salt/common/tools/sbin/so-fleet-user-add index 9d80c2076..8ce7325c3 100755 --- a/salt/common/tools/sbin/so-fleet-user-add +++ b/salt/common/tools/sbin/so-fleet-user-add @@ -41,10 +41,7 @@ if [[ $? == 0 ]]; then fi read -rs FLEET_PASS -if ! check_password "$FLEET_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." - exit 2 -fi +check_password_and_exit "$FLEET_PASS" FLEET_HASH=$(docker exec so-soctopus python -c "import bcrypt; print(bcrypt.hashpw('$FLEET_PASS'.encode('utf-8'), bcrypt.gensalt()).decode('utf-8'));" 2>&1) if [[ $? -ne 0 ]]; then diff --git a/salt/common/tools/sbin/so-thehive-user-add b/salt/common/tools/sbin/so-thehive-user-add index e26dc58fc..9cbe0cd56 100755 --- a/salt/common/tools/sbin/so-thehive-user-add +++ b/salt/common/tools/sbin/so-thehive-user-add @@ -41,10 +41,7 @@ if [[ $? == 0 ]]; then fi read -rs THEHIVE_PASS -if ! check_password "$THEHIVE_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." - exit 2 -fi +check_password_and_exit "$THEHIVE_PASS" # Create new user in TheHive resp=$(curl -sk -XPOST -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" -L "https://$THEHVIE_API_URL/user" -d "{\"login\" : \"$THEHIVE_USER\",\"name\" : \"$THEHIVE_USER\",\"roles\" : [\"read\",\"alert\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$THEHIVE_PASS\"}") diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index d7f9c2e6b..757ca10c1 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -98,7 +98,7 @@ function validatePassword() { if [[ $len -lt 6 ]]; then echo "Password does not meet the minimum requirements" exit 2 - fi + check_password_and_exit "$password" } function validateEmail() { From 649f339934628fed1452f84f6c727140bdc4d991 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Sep 2021 20:30:48 -0400 Subject: [PATCH 048/136] Correct typo --- salt/common/tools/sbin/so-user | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 757ca10c1..b3d65b128 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -98,6 +98,7 @@ function validatePassword() { if [[ $len -lt 6 ]]; then echo "Password does not meet the minimum requirements" exit 2 + fi check_password_and_exit "$password" } From 9fb28709d5b9b8f57ae36c776727ae32d8784870 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 3 Sep 2021 10:47:00 -0400 Subject: [PATCH 049/136] Add maxfiles to the steno config --- salt/pcap/files/config | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/pcap/files/config b/salt/pcap/files/config index 048775ef7..900234bc1 100644 --- a/salt/pcap/files/config +++ b/salt/pcap/files/config @@ -1,11 +1,13 @@ {%- set interface = salt['pillar.get']('sensor:interface', 'bond0') %} {%- set diskfreepercentage = salt['pillar.get']('steno:diskfreepercentage', 10) %} +{%- set maxfiles = salt['pillar.get']('steno:maxfiles', 30000) %} + { "Threads": [ { "PacketsDirectory": "/nsm/pcap" , "IndexDirectory": "/nsm/pcapindex" - , "MaxDirectoryFiles": 30000 + , "MaxDirectoryFiles": {{ maxfiles }} , "DiskFreePercentage": {{ diskfreepercentage }} } ] @@ -15,4 +17,4 @@ , "Host": "127.0.0.1" , "Flags": ["-v", "--uid=stenographer", "--gid=stenographer"{{ BPF_COMPILED }}] , "CertPath": "/etc/stenographer/certs" -} \ No newline at end of file +} From fbbb7f4e85fdc31427e83483b4062c8ee41cfa4d Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 3 Sep 2021 15:54:05 -0400 Subject: [PATCH 050/136] Add auditor role; update analyst role with correct syntax --- .../tools/sbin/so-elasticsearch-roles-load | 2 +- salt/elasticsearch/roles/analyst.json | 97 +++++++++++-------- 2 files changed, 58 insertions(+), 41 deletions(-) diff --git a/salt/common/tools/sbin/so-elasticsearch-roles-load b/salt/common/tools/sbin/so-elasticsearch-roles-load index 3b0f580fe..d094163ee 100644 --- a/salt/common/tools/sbin/so-elasticsearch-roles-load +++ b/salt/common/tools/sbin/so-elasticsearch-roles-load @@ -51,7 +51,7 @@ cd ${ELASTICSEARCH_ROLES} echo "Loading templates..." for role in *; do name=$(echo "$role" | cut -d. -f1) - so-elasticsearch-query security/roles/$name -XPUT -d @"$role" + so-elasticsearch-query _security/role/$name -XPUT -d @"$role" done cd - >/dev/null diff --git a/salt/elasticsearch/roles/analyst.json b/salt/elasticsearch/roles/analyst.json index f81c5a67e..3fbaa9489 100644 --- a/salt/elasticsearch/roles/analyst.json +++ b/salt/elasticsearch/roles/analyst.json @@ -1,45 +1,62 @@ { - "elasticsearch": { - "cluster": [ - "cancel_task", - "create_snapshot", - "monitor", - "monitor_data_frame_transforms", - "monitor_ml", - "monitor_rollup", - "monitor_snapshot", - "monitor_text_structure", - "monitor_transform", - "monitor_watcher", - "read_ccr", - "read_ilm", - "read_pipeline", - "read_slm" - ], - "indices": [ - { - "names": [ - "so-*" - ], - "privileges": [ - "read", - "read_cross_cluster", - "monitor", - "view_index_metadata" - ] - } - ], - "run_as": [] - }, - "kibana": [ + "cluster": [ + "cancel_task", + "create_snapshot", + "monitor", + "monitor_data_frame_transforms", + "monitor_ml", + "monitor_rollup", + "monitor_snapshot", + "monitor_text_structure", + "monitor_transform", + "monitor_watcher", + "read_ccr", + "read_ilm", + "read_pipeline", + "read_slm" + ], + "indices": [ { - "spaces": [ - "*" + "names": [ + "so-*" ], - "base": [ - "read" - ], - "feature": {} + "privileges": [ + "index", + "read", + "read_cross_cluster", + "monitor", + "view_index_metadata" + ] } - ] + ], + "applications": [ + { + "application": "kibana-.kibana", + "privileges": [ + "feature_discover.all", + "feature_dashboard.all", + "feature_canvas.all", + "feature_maps.all", + "feature_ml.all", + "feature_logs.read", + "feature_visualize.all", + "feature_infrastructure.read", + "feature_apm.read", + "feature_uptime.read", + "feature_siem.read", + "feature_dev_tools.read", + "feature_advancedSettings.read", + "feature_indexPatterns.read", + "feature_savedObjectsManagement.read", + "feature_savedObjectsTagging.read", + "feature_fleet.all", + "feature_actions.read", + "feature_stackAlerts.read" + ], + "resources": [ + "*" + ] + } + ], + "run_as": [] } \ No newline at end of file From 94ea1f856b4ce8cb1db30321a7a866de2ab70475 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 3 Sep 2021 15:59:48 -0400 Subject: [PATCH 051/136] Add auditor role; update analyst role with correct syntax --- salt/elasticsearch/roles/auditor.json | 59 +++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 salt/elasticsearch/roles/auditor.json diff --git a/salt/elasticsearch/roles/auditor.json b/salt/elasticsearch/roles/auditor.json new file mode 100644 index 000000000..26df1207c --- /dev/null +++ b/salt/elasticsearch/roles/auditor.json @@ -0,0 +1,59 @@ +{ + "cluster": [ + "monitor", + "monitor_data_frame_transforms", + "monitor_ml", + "monitor_rollup", + "monitor_snapshot", + "monitor_text_structure", + "monitor_transform", + "monitor_watcher", + "read_ccr", + "read_ilm", + "read_pipeline", + "read_slm" + ], + "indices": [ + { + "names": [ + "so-*" + ], + "privileges": [ + "read", + "read_cross_cluster", + "monitor", + "view_index_metadata" + ] + } + ], + "applications": [ + { + "application": "kibana-.kibana", + "privileges": [ + "feature_discover.read", + "feature_dashboard.read", + "feature_canvas.read", + "feature_maps.read", + "feature_ml.read", + "feature_logs.read", + "feature_visualize.read", + "feature_infrastructure.read", + "feature_apm.read", + "feature_uptime.read", + "feature_siem.read", + "feature_dev_tools.read", + "feature_advancedSettings.read", + "feature_indexPatterns.read", + "feature_savedObjectsManagement.read", + "feature_savedObjectsTagging.read", + "feature_fleet.read", + "feature_actions.read", + "feature_stackAlerts.read" + ], + "resources": [ + "*" + ] + } + ], + "run_as": [] +} \ No newline at end of file From b7591093cfbc0ba74ee95a6dbcc3705860496f9f Mon Sep 17 00:00:00 2001 From: Rob Waight <43173714+rwaight@users.noreply.github.com> Date: Sat, 4 Sep 2021 09:45:03 -0400 Subject: [PATCH 052/136] Add index sorting to so-common-template.json Add index sorting to so-common-template.json --- salt/elasticsearch/templates/so/so-common-template.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 777bf3f53..c9772b5db 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -8,6 +8,8 @@ "index.refresh_interval":"30s", "index.routing.allocation.require.box_type":"hot", "index.mapping.total_fields.limit": "1500", + "index.sort.field": "@timestamp", + "index.sort.order": "desc", "analysis": { "analyzer": { "es_security_analyzer": { From e3900606dc3c3fe3a83892f448cb96dab75d35ab Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Sat, 4 Sep 2021 10:42:18 -0400 Subject: [PATCH 053/136] Enable index sorting by default but allow it to be disabled --- salt/elasticsearch/templates/so/so-common-template.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index c9772b5db..4394ebb65 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -1,3 +1,4 @@ +{%- set INDEX_SORTING = salt['pillar.get']('elasticsearch:index_sorting', True) %} { "index_patterns": ["so-*"], "version":50001, @@ -8,8 +9,10 @@ "index.refresh_interval":"30s", "index.routing.allocation.require.box_type":"hot", "index.mapping.total_fields.limit": "1500", +{%- if INDEX_SORTING is sameas true %} "index.sort.field": "@timestamp", "index.sort.order": "desc", +{%- endif %} "analysis": { "analyzer": { "es_security_analyzer": { From 3c59579f99ae23398d3d1bf5f12a03e127de7469 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 7 Sep 2021 13:03:30 -0400 Subject: [PATCH 054/136] Add maintenance privilege for analysts to refresh indices --- salt/elasticsearch/roles/analyst.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/roles/analyst.json b/salt/elasticsearch/roles/analyst.json index 3fbaa9489..fc788e2f1 100644 --- a/salt/elasticsearch/roles/analyst.json +++ b/salt/elasticsearch/roles/analyst.json @@ -22,9 +22,10 @@ ], "privileges": [ "index", + "maintenance", + "monitor", "read", "read_cross_cluster", - "monitor", "view_index_metadata" ] } From b8600be0f102c351c6260b44748ad69c79da6b17 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 8 Sep 2021 12:12:09 -0400 Subject: [PATCH 055/136] Incude server.publicBaseUrl --- salt/kibana/etc/kibana.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml index 6fcafe68f..2819f301d 100644 --- a/salt/kibana/etc/kibana.yml +++ b/salt/kibana/etc/kibana.yml @@ -3,9 +3,11 @@ {%- set ES = salt['pillar.get']('manager:mainip', '') -%} {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- set URLBASE = salt['pillar.get']('global:url_base') %} server.name: kibana server.host: "0" server.basePath: /kibana +server.publicBaseUrl: https://{{ URLBASE }}/kibana elasticsearch.hosts: [ "https://{{ ES }}:9200" ] elasticsearch.ssl.verificationMode: none #kibana.index: ".kibana" From f06ab8b77d3659e99bd9cd25470b58f99c3a0262 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 9 Sep 2021 08:55:36 -0400 Subject: [PATCH 056/136] testing defaults.yaml --- salt/elasticsearch/defaults.yaml | 426 +++++++++++++++---------------- salt/elasticsearch/init.sls | 3 +- 2 files changed, 213 insertions(+), 216 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 7c506a9d2..b3db5aa0c 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -56,17 +56,7 @@ elasticsearch: close: 45 delete: 365 - - - persistent: - cluster: - remote: - default-cluster-name: - seeds: - - 127.0.0.1:9300 - transient: - {} - defaults: + cluster_settings: cluster: max_voting_config_exclusions: 10 auto_shrink_voting_configuration: true @@ -86,9 +76,9 @@ elasticsearch: read_only: false remote: node: - attr: + attr: "" initial_connect_timeout: 30s - connect: true + #connect: true #DEPRECATION connections_per_cluster: 3 follower_lag: timeout: 90000ms @@ -99,16 +89,16 @@ elasticsearch: allocation: node_concurrent_incoming_recoveries: 2 include: - _tier: + _tier: "" node_initial_primaries_recoveries: 4 same_shard: host: false - total_shards_per_node: -1 + total_shards_per_node: "-1" require: - _tier: - shard_state: - reroute: - priority: NORMAL + _tier: "" + #shard_state: #DEPRECATION + #reroute: #DEPRECATION + #priority: NORMAL #DEPRECATION type: balanced disk: threshold_enabled: true @@ -119,7 +109,7 @@ elasticsearch: low: 95% enable_for_single_data_node: false flood_stage.frozen: 95% - include_relocations: true + #include_relocations: true #DEPRECATION reroute_interval: 60s awareness: attributes: [] @@ -133,7 +123,7 @@ elasticsearch: cluster_concurrent_rebalance: 2 node_concurrent_recoveries: 2 exclude: - _tier: + _tier: "" indices: tombstones: size: 500 @@ -148,7 +138,7 @@ elasticsearch: publish: timeout: 30000ms info_timeout: 10000ms - name: default-cluster-name + name: {{ grains.host }} # Will change if true cluster fault_detection: leader_check: interval: 1000ms @@ -158,10 +148,10 @@ elasticsearch: interval: 1000ms timeout: 10000ms retry_count: 3 - join: - timeout: 60000ms + #join: #DEPRECATION + #timeout: 60000ms #DEPRECATION max_shards_per_node: 1000 - initial_master_nodes: [] + #initial_master_nodes: [] # ERROR setting [cluster.initial_master_nodes] is not allowed when [discovery.type] is set to [single-node] snapshot: info: max_concurrent_fetches: 5 @@ -176,9 +166,9 @@ elasticsearch: level: INFO bootstrap: memory_lock: false - system_call_filter: true + #system_call_filter: true #DEPRECATION ctrlhandler: true - processors: 8 + #processors: 8 #DEPRECATION ingest: user_agent: cache_size: 1000 @@ -198,14 +188,14 @@ elasticsearch: - 0.0.0.0 tcp: reuse_address: true - keep_count: -1 - connect_timeout: 30s - keep_interval: -1 + keep_count: "-1" + #connect_timeout: 30s #DEPRECATION + keep_interval: "-1" no_delay: true keep_alive: true - receive_buffer_size: -1b - keep_idle: -1 - send_buffer_size: -1b + receive_buffer_size: "-1b" + keep_idle: "-1" + send_buffer_size: "-1b" bind_host: - 0.0.0.0 server: true @@ -215,15 +205,15 @@ elasticsearch: overhead: 2.0 publish_host: - 0.0.0.0 - pidfile: + pidfile: "" path: data: [] logs: /var/log/elasticsearch - shared_data: + shared_data: "" home: /usr/share/elasticsearch repo: [] search: - default_search_timeout: -1 + default_search_timeout: "-1" highlight: term_vector_multi_value: true default_allow_partial_results: true @@ -234,10 +224,10 @@ elasticsearch: keep_alive_interval: 1m remote: node: - attr: - initial_connect_timeout: 30s - connect: true - connections_per_cluster: 3 + attr: "" + #initial_connect_timeout: 30s #DEPRECATION + #connect: true #DEPRECATION + #connections_per_cluster: 3 #DEPRECATION default_keep_alive: 5m max_keep_alive: 24h aggs: @@ -262,9 +252,9 @@ elasticsearch: wait_for_metadata_timeout: 60s repositories: fs: - compress: false + #compress: false #DEPRECATION chunk_size: 9223372036854775807b - location: + location: "" url: supported_protocols: - http @@ -297,32 +287,32 @@ elasticsearch: queue_capacity: 1024 max_concurrent_policy_executions: 50 xpack: - flattened: - enabled: true + #flattened: #DEPRECATION + #enabled: true #DEPRECATION watcher: execution: scroll: size: 0 - timeout: + timeout: "" default_throttle_period: 5s internal: ops: bulk: - default_timeout: + default_timeout: "" index: - default_timeout: + default_timeout: "" search: - default_timeout: + default_timeout: "" thread_pool: queue_size: 1000 size: 40 index: rest: - direct_access: + direct_access: "" use_ilm_index_management: true - history: - cleaner_service: - enabled: true + #history: #DEPRECATION + #cleaner_service: #DEPRECATION + #enabled: true #DEPRECATION trigger: schedule: ticker: @@ -330,11 +320,11 @@ elasticsearch: enabled: true input: search: - default_timeout: + default_timeout: "" encrypt_sensitive_data: false transform: search: - default_timeout: + default_timeout: "" stop: timeout: 30s watch: @@ -347,15 +337,15 @@ elasticsearch: actions: 1 actions: bulk: - default_timeout: + default_timeout: "" index: - default_timeout: - eql: - enabled: true - data_frame: - enabled: true - ilm: - enabled: true + default_timeout: "" + #eql: #DEPRECATION + #enabled: true #DEPRECATION + #data_frame: #DEPRECATION + #enabled: true #DEPRECATION + #ilm: #DEPRECATION + #enabled: true #DEPRECATION monitoring: migration: decommission_alerts: false @@ -390,7 +380,7 @@ elasticsearch: elasticsearch: collection: enabled: true - enabled: true + #enabled: true #DEPRECATION graph: enabled: true searchable: @@ -410,16 +400,16 @@ elasticsearch: min_time_delta: 60s decay: interval: 60s - size.max_headroom: -1 + size.max_headroom: "-1" range_size: 16mb max_freq: 100 rollup: - enabled: true + #enabled: true #DEPRECATION task_thread_pool: - queue_size: -1 + queue_size: "-1" size: 1 - sql: - enabled: true + #sql: #DEPRECATION + #enabled: true #DEPRECATION searchable_snapshots: cache_fetch_async_thread_pool: core: 0 @@ -439,13 +429,13 @@ elasticsearch: - trial self_generated: type: basic - logstash: - enabled: true + #logstash: #DEPRECATION + #enabled: true #DEPRECATION notification: pagerduty: - default_account: + default_account: "" email: - default_account: + default_account: "" html: sanitization: allow: @@ -464,9 +454,9 @@ elasticsearch: enabled: true interval: 15s jira: - default_account: + default_account: "" slack: - default_account: + default_account: "" security: operator_privileges: enabled: false @@ -484,6 +474,10 @@ elasticsearch: enabled: true ssl: enabled: true + verification_mode: none + certificate_authorities: /usr/share/elasticsearch/config/ca.crt + key: /usr/share/elasticsearch/config/elasticsearch.key + certificate: /usr/share/elasticsearch/config/elasticsearch.crt ssl: diagnose: trust: true @@ -532,13 +526,13 @@ elasticsearch: ttl: 24h delete: interval: 24h - timeout: -1 + timeout: "-1" enabled: false hashing: algorithm: pbkdf2 anonymous: authz_exception: true - roles: + roles: - superuser username: anonymous_user run_as: @@ -553,7 +547,7 @@ elasticsearch: token: delete: interval: 30m - timeout: -1 + timeout: "-1" enabled: false thread_pool: queue_size: 1000 @@ -571,13 +565,17 @@ elasticsearch: enabled: true ssl: enabled: true + client_authentication: none + key: /usr/share/elasticsearch/config/elasticsearch.key + certificate: /usr/share/elasticsearch/config/elasticsearch.crt + certificate_authorities: /usr/share/elasticsearch/config/ca.crt automata: max_determinized_states: 100000 cache: size: 10000 ttl: 48h enabled: true - user: null + user: "" authz: store: privileges: @@ -585,10 +583,10 @@ elasticsearch: ttl: 24h max_size: 10000 roles: - index: - cache: - ttl: 20m - max_size: 10000 + #index: #DEPRECATION + #cache: #DEPRECATION + #ttl: 20m #DEPRECATION + #max_size: 10000 #DEPRECATION cache: max_size: 10000 negative_lookup_cache: @@ -598,9 +596,9 @@ elasticsearch: max_size_in_bytes: 104857600 transform: num_transform_failure_retries: 10 - enabled: true - vectors: - enabled: true + #enabled: true #DEPRECATION + #vectors: #DEPRECATION + #enabled: true #DEPRECATION ccr: enabled: true ccr_thread_pool: @@ -608,14 +606,14 @@ elasticsearch: size: 32 idp: privileges: - application: + application: "" cache: size: 100 ttl: 90m metadata: signing: keystore: - alias: + alias: "" slo_endpoint: post: "https:" redirect: "https:" @@ -625,20 +623,20 @@ elasticsearch: allowed_nameid_formats: - urn:oasis:names:tc:SAML:2.0:nameid-format:transient contact: - given_name: - email: - surname: + given_name: "" + email: "" + surname: "" organization: - display_name: - name: + display_name: "" + name: "" url: "http:" sso_endpoint: post: "https:" redirect: "https:" - entity_id: + entity_id: "" signing: keystore: - alias: + alias: "" sp: cache: size: 1000 @@ -646,15 +644,15 @@ elasticsearch: wildcard: path: wildcard_services.json enabled: false - slm: - enabled: true - enrich: - enabled: true + #slm: + #enabled: true #DEPRECATION + #enrich: #DEPRECATION + #enabled: true #DEPRECATION http: default_connection_timeout: 10s proxy: - host: - scheme: + host: "" + scheme: "" port: 0 whitelist: - "*" @@ -673,7 +671,7 @@ elasticsearch: inference_model: cache_size: 40% time_to_live: 5m - nightly_maintenance_requests_per_second: -1.0 + nightly_maintenance_requests_per_second: "-1.0" node_concurrent_job_allocations: 2 max_model_memory_limit: 0b enabled: false @@ -713,15 +711,15 @@ elasticsearch: medium: 30s thread_pool: force_merge: - queue_size: -1 + queue_size: "-1" size: 1 fetch_shard_started: core: 1 max: 16 keep_alive: 5m - listener: - queue_size: -1 - size: 4 + #listener: #DEPRECATION + #queue_size: "-1" #DEPRECATION + #size: 4 #DEPRECATION refresh: core: 1 max: 4 @@ -738,12 +736,12 @@ elasticsearch: max: 4 keep_alive: 5m search: - max_queue_size: 1000 + #max_queue_size: 1000 #DEPRECATION queue_size: 1000 size: 13 - auto_queue_frame_size: 2000 - target_response_time: 1s - min_queue_size: 1000 + #auto_queue_frame_size: 2000 #DEPRECATION + #target_response_time: 1s #DEPRECATION + #min_queue_size: 1000 #DEPRECATION fetch_shard_store: core: 1 max: 16 @@ -774,23 +772,23 @@ elasticsearch: max: 4 keep_alive: 5m search_throttled: - max_queue_size: 100 + #max_queue_size: 100 #DEPRECATION queue_size: 100 size: 1 - auto_queue_frame_size: 200 - target_response_time: 1s - min_queue_size: 100 + #auto_queue_frame_size: 200 #DEPRECATION + #target_response_time: 1s #DEPRECATION + #min_queue_size: 100 #DEPRECATION index: codec: default recovery: - type: + type: "" store: - type: + type: "" fs: fs_lock: native preload: [] snapshot: - uncached_chunk_size: -1b + uncached_chunk_size: "-1b" cache: excluded_file_types: [] monitor: @@ -821,21 +819,21 @@ elasticsearch: transport: tcp: reuse_address: true - keep_count: -1 - connect_timeout: 30s - keep_interval: -1 - compress: false - port: 9300-9400 + keep_count: "-1" + #connect_timeout: 30s #DEPRECATION + keep_interval: "-1" + #compress: false #DEPRECATION + #port: 9300-9400 #DEPRECATION no_delay: true keep_alive: true - receive_buffer_size: -1b - keep_idle: -1 - send_buffer_size: -1b + receive_buffer_size: "-1b" + keep_idle: "-1" + send_buffer_size: "-1b" bind_host: - 0.0.0.0 connect_timeout: 30s compress: false - ping_schedule: -1 + ping_schedule: "-1" connections_per_node: recovery: 2 state: 1 @@ -856,7 +854,7 @@ elasticsearch: port: 9300-9400 host: [] publish_port: 9300 - tcp_no_delay: true + #tcp_no_delay: true #DEPRECATION publish_host: {{ grains.host }} netty: receive_predictor_size: 64kb @@ -865,57 +863,57 @@ elasticsearch: receive_predictor_min: 64kb boss_count: 1 script: - allowed_contexts: [] - max_compilations_rate: 20000/1m - cache: - max_size: 100 - expire: 0ms + allowed_contexts: none # ERROR have to set to none - should be list + #max_compilations_rate: 20000/1m #DEPRECATION + #cache: #DEPRECATION + #max_size: 100 #DEPRECATION + #expire: 0ms #DEPRECATION painless: regex: enabled: limited limit-factor: 6 max_size_in_bytes: 65535 - allowed_types: [] + allowed_types: none # ERROR have to set to none - should be list disable_max_compilations_rate: false indexing_pressure: memory: limit: 10% node: - data: true - roles: - - data_frozen - - data_warm - - transform - - data - - remote_cluster_client - - data_cold - - data_content - - data_hot - - ingest - - master - max_local_storage_nodes: 1 + #data: true #DEPRECATION + # roles: + # - data_frozen + # - data_warm + # - transform ERROR + # - data + # - remote_cluster_client + # - data_cold + # - data_content + # - data_hot + # - ingest + # - master + #max_local_storage_nodes: 1 #DEPRECATION processors: 8 store: allow_mmap: true - ingest: true - master: true - pidfile: - transform: true - remote_cluster_client: true + #ingest: true #DEPRECATION + #master: true #DEPRECATION + pidfile: "" + #transform: true #DEPRECATION + #remote_cluster_client: true #DEPRECATION enable_lucene_segment_infos_trace: false - local_storage: true + #local_storage: true #DEPRECATION name: {{ grains.host }} id: seed: 0 - voting_only: false + #voting_only: false #DEPRECATION attr: - transform: - node: true + #transform: ERROR + # node: true ERROR xpack: - installed: true + installed: "" box_type: hot portsfile: false - ml: true + #ml: true #DEPRECATION indices: replication: retry_timeout: 60s @@ -927,7 +925,7 @@ elasticsearch: max_in_flight_updates: 10 memory: interval: 5s - max_index_buffer_size: -1 + max_index_buffer_size: "-1" shard_inactive_time: 5m index_buffer_size: 10% min_index_buffer_size: 48mb @@ -968,7 +966,7 @@ elasticsearch: requests: cache: size: 1% - expire: 0ms + expire: 1ms #0ms - ERROR when set to 0ms, set to 1ms and ERROR gone store: delete: shard: @@ -990,7 +988,7 @@ elasticsearch: master_timeout: 30s fielddata: cache: - size: -1b + size: "-1b" plugin: mandatory: [] slm: @@ -999,48 +997,48 @@ elasticsearch: retention_duration: 1h history_index_enabled: true discovery: - seed_hosts: [] + #seed_hosts: [] # ERROR - it is forbidden to set both [discovery.seed_hosts] and [discovery.zen.ping.unicast.hosts] unconfigured_bootstrap_timeout: 3s request_peers_timeout: 3000ms zen: - commit_timeout: 30s - no_master_block: write - join_retry_delay: 100ms - join_retry_attempts: 3 - ping: - unicast: - concurrent_connects: 10 - hosts: [] - hosts.resolve_timeout: 5s - master_election: - ignore_non_master_pings: false - wait_for_joins_timeout: 30000ms - send_leave_request: true + #commit_timeout: 30s #DEPRECATION + #no_master_block: write #DEPRECATION + #join_retry_delay: 100ms #DEPRECATION + #join_retry_attempts: 3 #DEPRECATION + #ping: + #unicast: + #concurrent_connects: 10 # ERROR forbidden to set both [discovery.seed_resolver.max_concurrent_resolvers] and [discovery.zen.ping.unicast.concurrent_connects] + #hosts: [] # ERROR - it is forbidden to set both [discovery.seed_hosts] and [discovery.zen.ping.unicast.hosts] + #hosts.resolve_timeout: 5s # ERROR forbidden to set both [discovery.seed_resolver.timeout] and [discovery.zen.ping.unicast.hosts.resolve_timeout] + #master_election: #DEPRECATION + #ignore_non_master_pings: false #DEPRECATION + #wait_for_joins_timeout: 30000ms #DEPRECATION + #send_leave_request: true #DEPRECATION ping_timeout: 3s - bwc_ping_timeout: 3s - join_timeout: 60000ms - publish_diff: - enable: true - publish: - max_pending_cluster_states: 25 - minimum_master_nodes: -1 - unsafe_rolling_upgrades_enabled: true - hosts_provider: [] - publish_timeout: 30s - fd: - connect_on_network_disconnect: false - ping_interval: 1s - ping_retries: 3 - register_connection_listener: true - ping_timeout: 30s - max_pings_from_another_master: 3 + #bwc_ping_timeout: 3s #DEPRECATION + #join_timeout: 60000ms #DEPRECATION + #publish_diff: + #enable: true #DEPRECATION + #publish: #DEPRECATION + #max_pending_cluster_states: 25 #DEPRECATION + #minimum_master_nodes: "-1" #DEPRECATION + #unsafe_rolling_upgrades_enabled: true #DEPRECATION + #hosts_provider: # ERROR forbidden to set both [discovery.seed_providers] and [discovery.zen.hosts_provider] has to be commented out + #publish_timeout: 30s #DEPRECATION + #fd: #DEPRECATION + #connect_on_network_disconnect: false #DEPRECATION + #ping_interval: 1s #DEPRECATION + #ping_retries: 3 #DEPRECATION + #register_connection_listener: true #DEPRECATION + #ping_timeout: 30s #DEPRECATION + #max_pings_from_another_master: 3 #DEPRECATION initial_state_timeout: 30s cluster_formation_warning_timeout: 10000ms - seed_providers: [] - type: single-node - seed_resolver: - max_concurrent_resolvers: 10 - timeout: 5s + #seed_providers: # ERROR forbidden to set both [discovery.seed_providers] and [discovery.zen.hosts_provider] has to be commented out + type: zen # ERROR java.lang.IllegalArgumentException: node with [discovery.type] set to [single-node] must be master-eligible # test turning off + #seed_resolver: + #max_concurrent_resolvers: 10 # ERROR forbidden to set both [discovery.seed_resolver.max_concurrent_resolvers] and [discovery.zen.ping.unicast.concurrent_connects] + #timeout: 5s # forbidden to set both [discovery.seed_resolver.timeout] and [discovery.zen.ping.unicast.hosts.resolve_timeout] find_peers_interval: 1000ms probe: connect_timeout: 30s @@ -1048,7 +1046,7 @@ elasticsearch: http: cors: max-age: 1728000 - allow-origin: + allow-origin: "" allow-headers: X-Requested-With,Content-Type,Content-Length allow-credentials: false allow-methods: OPTIONS,HEAD,GET,POST,PUT,DELETE @@ -1060,10 +1058,10 @@ elasticsearch: pipelining: max_events: 10000 type.default: netty4 - content_type: - required: true + #content_type: #DEPRECATION + #required: true #DEPRECATION host: [] - publish_port: -1 + publish_port: "-1" read_timeout: 0ms max_content_length: 100mb netty: @@ -1072,40 +1070,40 @@ elasticsearch: worker_count: 0 tcp: reuse_address: true - keep_count: -1 - keep_interval: -1 + keep_count: "-1" + keep_interval: "-1" no_delay: true keep_alive: true - receive_buffer_size: -1b - keep_idle: -1 - send_buffer_size: -1b + receive_buffer_size: "-1b" + keep_idle: "-1" + send_buffer_size: "-1b" bind_host: [] client_stats: enabled: true reset_cookies: false - max_warning_header_count: -1 + max_warning_header_count: "-1" tracer: include: [] exclude: [] - max_warning_header_size: -1b + max_warning_header_size: "-1b" detailed_errors: enabled: true port: 9200-9300 max_header_size: 8kb - tcp_no_delay: true + #tcp_no_delay: true #DEPRECATION compression: false publish_host: [] gateway: - recover_after_master_nodes: 0 - expected_nodes: -1 - recover_after_data_nodes: -1 - expected_data_nodes: -1 + #recover_after_master_nodes: 0 #DEPRECATION + #expected_nodes: "-1" #DEPRECATION + recover_after_data_nodes: "-1" + expected_data_nodes: "-1" write_dangling_indices_info: true slow_write_logging_threshold: 10s recover_after_time: 0ms - expected_master_nodes: -1 - recover_after_nodes: -1 - auto_import_dangling_indices: false + #expected_master_nodes: "-1" #DEPRECATION + #recover_after_nodes: "-1" #DEPRECATION + #auto_import_dangling_indices: false #DEPRECATION snapshot: refresh_repo_uuid_on_restore: true max_concurrent_operations: 1000 diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 49f9fc04c..2deeef55c 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -140,10 +140,9 @@ eslog4jfile: esyml: file.managed: - name: /opt/so/conf/elasticsearch/elasticsearch.yml - - source: salt://elasticsearch/files/elasticsearch.yml + - contents: {{ ESCONFIG.elasticsearch.cluster_settings | yaml }} - user: 930 - group: 939 - - template: jinja esyml_test: file.managed: From 93f2cd75a4c540e289305a8c712783ebb498b5aa Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 9 Sep 2021 10:19:46 -0400 Subject: [PATCH 057/136] add the jinja template --- salt/elasticsearch/files/elasticsearch.yaml.jinja | 1 + salt/elasticsearch/init.sls | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 salt/elasticsearch/files/elasticsearch.yaml.jinja diff --git a/salt/elasticsearch/files/elasticsearch.yaml.jinja b/salt/elasticsearch/files/elasticsearch.yaml.jinja new file mode 100644 index 000000000..caf13173f --- /dev/null +++ b/salt/elasticsearch/files/elasticsearch.yaml.jinja @@ -0,0 +1 @@ +{{ ESCONFIG | yaml(False) }} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 2deeef55c..7e59f5c43 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -140,9 +140,13 @@ eslog4jfile: esyml: file.managed: - name: /opt/so/conf/elasticsearch/elasticsearch.yml - - contents: {{ ESCONFIG.elasticsearch.cluster_settings | yaml }} + - source: salt://elasticsearch/files/elasticsearch.yaml.jinja - user: 930 - group: 939 + - defaults: + ESCONFIG: {{ ESCONFIG.elasticsearch.cluster_settings }} + - template: jinja + esyml_test: file.managed: From 5d235e932c02a2f3b06771c42c09e73cf5fc080b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Sep 2021 10:46:28 -0400 Subject: [PATCH 058/136] Fix Raid Status for cloud --- salt/common/tools/sbin/so-raid-status | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/common/tools/sbin/so-raid-status b/salt/common/tools/sbin/so-raid-status index e817554d3..54c2b3024 100755 --- a/salt/common/tools/sbin/so-raid-status +++ b/salt/common/tools/sbin/so-raid-status @@ -20,6 +20,10 @@ appliance_check() { {%- if salt['grains.get']('sosmodel', '') %} APPLIANCE=1 + {%- if grains['sosmodel'] in ['SO2AMI01', 'SO2GCI01', 'SO2AZI01'] %} + echo "nsmraid=0" > /opt/so/log/raid/status.log + exit 0 + {%- endif %} DUDEYOUGOTADELL=$(dmidecode |grep Dell) if [[ -n $DUDEYOUGOTADELL ]]; then APPTYPE=dell From c83f119cc08124dc41859ade5e01e00ba08e4bf6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Sep 2021 10:59:35 -0400 Subject: [PATCH 059/136] Update so-raid-status --- salt/common/tools/sbin/so-raid-status | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/common/tools/sbin/so-raid-status b/salt/common/tools/sbin/so-raid-status index 54c2b3024..4664a5961 100755 --- a/salt/common/tools/sbin/so-raid-status +++ b/salt/common/tools/sbin/so-raid-status @@ -21,7 +21,6 @@ appliance_check() { {%- if salt['grains.get']('sosmodel', '') %} APPLIANCE=1 {%- if grains['sosmodel'] in ['SO2AMI01', 'SO2GCI01', 'SO2AZI01'] %} - echo "nsmraid=0" > /opt/so/log/raid/status.log exit 0 {%- endif %} DUDEYOUGOTADELL=$(dmidecode |grep Dell) From 63f67b35009204606a5b30894d9629f89912cd61 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 9 Sep 2021 14:16:05 -0400 Subject: [PATCH 060/136] Rephrase screen that warns about more RAM requirements --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 125335729..c7954e65f 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -114,7 +114,7 @@ whiptail_components_adv_warning() { [ -n "$TESTING" ] && return - whiptail --title "$whiptail_title" --msgbox "Please keep in mind the more services that you enable the more RAM that is required." 8 75 + whiptail --title "$whiptail_title" --msgbox "The next screen will provide the ability to choose optional services to be enabled for this installation. Be aware that the more services you enable the more RAM that is required." 8 75 } whiptail_create_admin_user() { From 94688a9adb4ba7fa2ddee20eb0c5d9b915658a30 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 9 Sep 2021 14:29:09 -0400 Subject: [PATCH 061/136] Eliminate adv component popup --- setup/so-setup | 1 - setup/so-whiptail | 12 +++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 958d8aea1..7d5ae0a0e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -473,7 +473,6 @@ if [[ $is_manager && ! $is_eval ]]; then fi if [[ $is_manager ]]; then - whiptail_components_adv_warning whiptail_enable_components if [[ "$STRELKA" = 1 ]]; then diff --git a/setup/so-whiptail b/setup/so-whiptail index c7954e65f..1f0986119 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -110,13 +110,6 @@ whiptail_check_exitstatus() { esac } -whiptail_components_adv_warning() { - - [ -n "$TESTING" ] && return - - whiptail --title "$whiptail_title" --msgbox "The next screen will provide the ability to choose optional services to be enabled for this installation. Be aware that the more services you enable the more RAM that is required." 8 75 -} - whiptail_create_admin_user() { [ -n "$TESTING" ] && return @@ -408,9 +401,10 @@ whiptail_enable_components() { PLAYBOOK=0 STRELKA=0 +description="Choose optional services to be enabled for this installation. Be aware that the more services you enable the more RAM that is required." if [[ $is_eval ]]; then COMPONENTS=$(whiptail --title "$whiptail_title" --checklist \ - "Select Components to install:" 20 75 8 \ + "$description" 20 75 8 \ GRAFANA "Enable Grafana for system monitoring" ON \ OSQUERY "Enable Fleet with osquery" ON \ WAZUH "Enable Wazuh" ON \ @@ -419,7 +413,7 @@ if [[ $is_eval ]]; then STRELKA "Enable Strelka" ON 3>&1 1>&2 2>&3) else COMPONENTS=$(whiptail --title "$whiptail_title" --checklist \ - "Select Components to install:" 20 75 7 \ + "$description" 20 75 7 \ OSQUERY "Enable Fleet with osquery" ON \ WAZUH "Enable Wazuh" ON \ THEHIVE "Enable TheHive" ON \ From db50ef71b4297605b4d1eb7daef2b41c0d4c5cb6 Mon Sep 17 00:00:00 2001 From: doug Date: Fri, 10 Sep 2021 06:19:16 -0400 Subject: [PATCH 062/136] FEATURE: Add docs link to Setup #5459 --- setup/so-whiptail | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 1f0986119..ed4067da1 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1867,8 +1867,9 @@ whiptail_you_sure() { read -r -d '' you_sure_text <<- EOM Welcome to Security Onion Setup! - You can use Setup for lots of different use cases from a small standalone installation to a large distributed deployment for your enterprise. - + You can use Setup for lots of different use cases from a small standalone installation to a large distributed deployment for your enterprise. Don't forget to review the documentation at: + https://docs.securityonion.net + Setup uses keyboard navigation and you can use arrow keys to move around. Certain screens may provide a list and ask you to select one or more items from that list. You can use [SPACE] to select items and [ENTER] to proceed to the next screen. Would you like to continue? From 4f6a3269cb0e5c8c1378cf46c7b84bc389722b45 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 10 Sep 2021 09:59:47 -0400 Subject: [PATCH 063/136] Add more detail to syscollector --- salt/soc/files/soc/hunt.eventfields.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/files/soc/hunt.eventfields.json b/salt/soc/files/soc/hunt.eventfields.json index e519e78b6..7964a360b 100644 --- a/salt/soc/files/soc/hunt.eventfields.json +++ b/salt/soc/files/soc/hunt.eventfields.json @@ -45,7 +45,7 @@ ":kibana:": ["soc_timestamp", "host.name", "message", "kibana.log.meta.req.headers.x-real-ip", "event.dataset" ], "::rootcheck": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ], "::ossec": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ], - "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ], + "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "log.full", "event.dataset", "event.module" ], ":syslog:syslog": ["soc_timestamp", "host.name", "metadata.ip_address", "real_message", "syslog.priority", "syslog.application" ], ":aws:": ["soc_timestamp", "aws.cloudtrail.event_category", "aws.cloudtrail.event_type", "event.provider", "event.action", "event.outcome", "cloud.region", "user.name", "source.ip", "source.geo.region_iso_code" ], ":squid:": ["soc_timestamp", "url.original", "destination.ip", "destination.geo.country_iso_code", "user.name", "source.ip" ] From 9c310de459eeafab4913a8dd0533a53b0a5c5a17 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 10 Sep 2021 13:15:09 -0400 Subject: [PATCH 064/136] Add `--force` flag to idstools-rulecat under so-rule-update This forces idstools to pull from the url each time, which prevents it from clearing all.rules if idstools-rulecat is run twice within 15 minutes by any method (either restarting the container or running so-rule-update) --- salt/common/tools/sbin/so-rule-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-rule-update b/salt/common/tools/sbin/so-rule-update index 397719d61..a4e4d3cd7 100755 --- a/salt/common/tools/sbin/so-rule-update +++ b/salt/common/tools/sbin/so-rule-update @@ -10,4 +10,4 @@ got_root() { } got_root -docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat $1" +docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force $1" From 0b42b19763093f3597a6905138571a0a5777d6c7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 10 Sep 2021 13:41:58 -0400 Subject: [PATCH 065/136] Update so-rule-update to source so-common --- salt/common/tools/sbin/so-rule-update | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/so-rule-update b/salt/common/tools/sbin/so-rule-update index a4e4d3cd7..0fc691d36 100755 --- a/salt/common/tools/sbin/so-rule-update +++ b/salt/common/tools/sbin/so-rule-update @@ -1,13 +1,7 @@ #!/bin/bash -got_root() { - # Make sure you are root - if [ "$(id -u)" -ne 0 ]; then - echo "This script must be run using sudo!" - exit 1 - fi +. /usr/sbin/so-common -} got_root docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force $1" From 679faddd52469395a2b0ece8ab64c24cd67584ba Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 10 Sep 2021 13:44:37 -0400 Subject: [PATCH 066/136] Update so-rule-update to pass all args to docker exec Instead of passing $1, build a string from all args and add that to the command string for the docker exec statement --- salt/common/tools/sbin/so-rule-update | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-rule-update b/salt/common/tools/sbin/so-rule-update index 0fc691d36..a3c2616a4 100755 --- a/salt/common/tools/sbin/so-rule-update +++ b/salt/common/tools/sbin/so-rule-update @@ -2,6 +2,9 @@ . /usr/sbin/so-common +argstr="" +for arg in "$@"; do + argstr="${argstr} \"${arg}\"" +done -got_root -docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force $1" +docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}" From 0ae09cc6309ca237e4febf2f1425c1fca0405d92 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 13 Sep 2021 09:49:56 -0400 Subject: [PATCH 067/136] config changes --- salt/elasticsearch/defaults.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index b3db5aa0c..d84349963 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -863,8 +863,8 @@ elasticsearch: receive_predictor_min: 64kb boss_count: 1 script: - allowed_contexts: none # ERROR have to set to none - should be list - #max_compilations_rate: 20000/1m #DEPRECATION + #allowed_contexts: none # ERROR have to set to none - should be list + max_compilations_rate: 20000/1m #DEPRECATION #cache: #DEPRECATION #max_size: 100 #DEPRECATION #expire: 0ms #DEPRECATION @@ -873,7 +873,7 @@ elasticsearch: enabled: limited limit-factor: 6 max_size_in_bytes: 65535 - allowed_types: none # ERROR have to set to none - should be list + #allowed_types: none # ERROR have to set to none - should be list disable_max_compilations_rate: false indexing_pressure: memory: From f8ab0ac8a908359432a2a95cf7649c604ad120b1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 13 Sep 2021 15:04:39 -0400 Subject: [PATCH 068/136] config changes --- salt/elasticsearch/defaults.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index d84349963..d7f21b88e 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -532,9 +532,9 @@ elasticsearch: algorithm: pbkdf2 anonymous: authz_exception: true - roles: - - superuser - username: anonymous_user + #roles: + # - superuser + username: _anonymous run_as: enabled: true reserved_realm: @@ -966,7 +966,7 @@ elasticsearch: requests: cache: size: 1% - expire: 1ms #0ms - ERROR when set to 0ms, set to 1ms and ERROR gone + #expire: 1ms #0ms - ERROR when set to 0ms, set to 1ms and ERROR gone store: delete: shard: From 0b0d5085853ffba59e57f103eb23cdb34500b1e8 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 14 Sep 2021 12:01:14 -0400 Subject: [PATCH 069/136] so-import-evtx - tweaks --- salt/common/tools/sbin/so-import-evtx | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin/so-import-evtx index fe53ee601..e860881f3 100644 --- a/salt/common/tools/sbin/so-import-evtx +++ b/salt/common/tools/sbin/so-import-evtx @@ -46,7 +46,7 @@ function evtx2es() { so-evtx2es02 \ --host {{ MANAGERIP }} --scheme https \ --index so-beats-$INDEX_DATE --pipeline import.wel \ - --login {{ES_USER}} --pwd {{ES_PW}} "/tmp/$RUNID.evtx" 2>&1 + --login {{ES_USER}} --pwd {{ES_PW}} "/tmp/$RUNID.evtx" 1>/dev/null 2>/dev/null docker run --rm \ @@ -75,7 +75,7 @@ for i in "$@"; do fi done -# track if we have any valid or invalid pcaps +# track if we have any valid or invalid evtx INVALID_EVTXS="no" VALID_EVTXS="no" @@ -108,14 +108,10 @@ for EVTX in "$@"; do EVTX_DIR=$HASH_DIR/evtx mkdir -p $EVTX_DIR - # generate IDS alerts and write them to standard pipeline + # import evtx and write them to import ingest pipeline echo "- importing logs with evtx2es" evtx2es "${EVTX}" $HASH - #START=$(pcapinfo "${EVTX}" -a |grep "First packet time:" | awk '{print $4}') - #END=$(pcapinfo "${EVTX}" -e |grep "Last packet time:" | awk '{print $4}') - #echo "- saving EVTX data spanning dates $START through $END" - # compare $START to $START_OLDEST START=$(cat /nsm/import/evtx-start_oldest) START_COMPARE=$(date -d $START +%s) From 74b0b365bdd7ce28206df6675686008ea3d86537 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 14 Sep 2021 13:23:07 -0400 Subject: [PATCH 070/136] Fleet SA - SOC Link Fix --- salt/soc/files/soc/soc.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 6119f0e6b..4798ef31e 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -3,7 +3,8 @@ {%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %} {%- set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %} {%- set THEHIVE = salt['pillar.get']('manager:thehive', '0') %} -{%- set OSQUERY = salt['pillar.get']('manager:osquery', '0') %} +{% set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) %} +{% set FLEETNODE = salt['pillar.get']('global:fleet_node', False) %} {%- set GRAFANA = salt['pillar.get']('manager:grafana', '0') %} {%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %} {%- set API_TIMEOUT = salt['pillar.get']('sensoroni:api_timeout_ms', 0) %} @@ -116,7 +117,7 @@ {%- if THEHIVE == 0 %} "toolTheHive", {%- endif %} - {%- if OSQUERY == 0 %} + {%- if FLEETMANAGER or FLEETNODE %} "toolFleet", {%- endif %} {%- if GRAFANA == 0 %} From 614a6dc9fe46db0f75da51408e2703c14c426abd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 14 Sep 2021 13:56:43 -0400 Subject: [PATCH 071/136] Update manager.sls --- pillar/elasticsearch/manager.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pillar/elasticsearch/manager.sls b/pillar/elasticsearch/manager.sls index 9ff97de5b..84ff89a23 100644 --- a/pillar/elasticsearch/manager.sls +++ b/pillar/elasticsearch/manager.sls @@ -1,7 +1,7 @@ elasticsearch: templates: - so/so-beats-template.json.jinja - - so/so-common-template.json + - so/so-common-template.json.jinja - so/so-firewall-template.json.jinja - so/so-flow-template.json.jinja - so/so-ids-template.json.jinja From 2cc25587d98f2cc469589aab540d3b62efae59a1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 14 Sep 2021 13:57:04 -0400 Subject: [PATCH 072/136] Update eval.sls --- pillar/elasticsearch/eval.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pillar/elasticsearch/eval.sls b/pillar/elasticsearch/eval.sls index 2dbb08f59..84ff89a23 100644 --- a/pillar/elasticsearch/eval.sls +++ b/pillar/elasticsearch/eval.sls @@ -1,7 +1,7 @@ elasticsearch: templates: - so/so-beats-template.json.jinja - - so/so-common-template.json + - so/so-common-template.json.jinja - so/so-firewall-template.json.jinja - so/so-flow-template.json.jinja - so/so-ids-template.json.jinja @@ -10,4 +10,4 @@ elasticsearch: - so/so-ossec-template.json.jinja - so/so-strelka-template.json.jinja - so/so-syslog-template.json.jinja - - so/so-zeek-template.json.jinja \ No newline at end of file + - so/so-zeek-template.json.jinja From 6ae2fba71fa50a5e1cf3fd578562bcfbfa7dff36 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 14 Sep 2021 13:57:26 -0400 Subject: [PATCH 073/136] Update search.sls --- pillar/elasticsearch/search.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pillar/elasticsearch/search.sls b/pillar/elasticsearch/search.sls index 9ff97de5b..84ff89a23 100644 --- a/pillar/elasticsearch/search.sls +++ b/pillar/elasticsearch/search.sls @@ -1,7 +1,7 @@ elasticsearch: templates: - so/so-beats-template.json.jinja - - so/so-common-template.json + - so/so-common-template.json.jinja - so/so-firewall-template.json.jinja - so/so-flow-template.json.jinja - so/so-ids-template.json.jinja From 2ffb723bbd77c3669687b13a42d8029e8b2d98a7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 14 Sep 2021 13:58:45 -0400 Subject: [PATCH 074/136] Rename so-common-template.json to so-common-template.json.jinja --- .../so/{so-common-template.json => so-common-template.json.jinja} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename salt/elasticsearch/templates/so/{so-common-template.json => so-common-template.json.jinja} (100%) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json.jinja similarity index 100% rename from salt/elasticsearch/templates/so/so-common-template.json rename to salt/elasticsearch/templates/so/so-common-template.json.jinja From ff989b1c736b5712bf5b1031684f2295ba1e0319 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 14 Sep 2021 14:03:00 -0400 Subject: [PATCH 075/136] Include wording in so-user relating to optional role parameter --- salt/common/tools/sbin/so-user | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index b3d65b128..7d4c55453 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -26,7 +26,7 @@ if [[ $# -lt 1 || $# -gt 3 ]]; then echo " where is one of the following:" echo "" echo " list: Lists all user email addresses currently defined in the identity system" - echo " add: Adds a new user to the identity system; requires 'email' parameter" + echo " add: Adds a new user to the identity system; requires 'email' parameter, while 'role' parameter is optional and defaults to $DEFAULT_ROLE" echo " addrole: Grants a role to an existing user; requires 'email' and 'role' parameters" echo " delrole: Removes a role from an existing user; requires 'email' and 'role' parameters" echo " update: Updates a user's password; requires 'email' parameter" From 9970e54081cf06711679817408f48a0c3556ffb2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 14 Sep 2021 14:03:22 -0400 Subject: [PATCH 076/136] Adjust custom_role examples to be more realistic --- salt/soc/files/soc/custom_roles | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/salt/soc/files/soc/custom_roles b/salt/soc/files/soc/custom_roles index 80ae7b147..b95b94da4 100644 --- a/salt/soc/files/soc/custom_roles +++ b/salt/soc/files/soc/custom_roles @@ -9,12 +9,15 @@ # Syntax => prebuiltRoleX: customRoleY: op # Explanation => roleY and roleZ are adjusted permissions of roleX, op is: # + add the new permissions/role mappings (default) -# - remove existing prebuilt permissions +# - remove existing "explicit" prebuilt permissions. This +# does not work with implictly inherited permissions. # -# In the example below, we will define a new role for junior analysts, -# that is nearly identical to the analyst role that comes with SOC, with the -# exception that it removes their ability to obtain details about other -# analysts in the system. +# In the example below, we will define two new roles for segregating +# analysts into two regions. Then we will remove the ability for all +# analysts to see the roles of other analysts. (Seperately we will need to +# define these two new roles in Elasticsearch so that each analyst region +# can only see data from their specific region's indices, but that is out +# of scope from this file.) # -# analyst: jr_analyst -# user-monitor: jr_analyst:- +# analyst: westcoast_analyst, eastcoast_analyst +# roles/read: user-monitor:- \ No newline at end of file From 551dba955ca6cdbfbca6eff9a5aea7483a21eedb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 09:20:33 -0400 Subject: [PATCH 077/136] set roles empty list --- salt/elasticsearch/defaults.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index d7f21b88e..bfe0e150b 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -532,8 +532,7 @@ elasticsearch: algorithm: pbkdf2 anonymous: authz_exception: true - #roles: - # - superuser + roles: [] username: _anonymous run_as: enabled: true From e0a289182f643556fd031f6cfd8c26c07f90c3ed Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 15 Sep 2021 09:28:23 -0400 Subject: [PATCH 078/136] Fix Fleet Link Logic --- salt/soc/files/soc/soc.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 4798ef31e..e9dfa50d8 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -117,7 +117,7 @@ {%- if THEHIVE == 0 %} "toolTheHive", {%- endif %} - {%- if FLEETMANAGER or FLEETNODE %} + {%- if not FLEETMANAGER and not FLEETNODE %} "toolFleet", {%- endif %} {%- if GRAFANA == 0 %} From 5c8067728e12fb2fec697252338ef42d6b3ebb8f Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 15 Sep 2021 10:22:17 -0400 Subject: [PATCH 079/136] Remove unnecessary logic --- salt/common/tools/sbin/so-yara-update | 114 ++++---------------------- 1 file changed, 17 insertions(+), 97 deletions(-) diff --git a/salt/common/tools/sbin/so-yara-update b/salt/common/tools/sbin/so-yara-update index 9f749727f..025cf8c70 100755 --- a/salt/common/tools/sbin/so-yara-update +++ b/salt/common/tools/sbin/so-yara-update @@ -1,5 +1,4 @@ #!/bin/bash - # Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # # This program is free software: you can redistribute it and/or modify @@ -20,13 +19,8 @@ echo "Starting to check for yara rule updates at $(date)..." output_dir="/opt/so/saltstack/default/salt/strelka/rules" mkdir -p $output_dir - repos="$output_dir/repos.txt" -ignorefile="$output_dir/ignore.txt" - -deletecounter=0 newcounter=0 -updatecounter=0 {% if ISAIRGAP is sameas true %} @@ -35,58 +29,21 @@ echo "Airgap mode enabled." clone_dir="/nsm/repo/rules/strelka" repo_name="signature-base" mkdir -p /opt/so/saltstack/default/salt/strelka/rules/signature-base - +# Ensure a copy of the license is available for the rules [ -f $clone_dir/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name # Copy over rules for i in $(find $clone_dir/yara -name "*.yar*"); do rule_name=$(echo $i | awk -F '/' '{print $NF}') - repo_sum=$(sha256sum $i | awk '{print $1}') - - # Check rules against those in ignore list -- don't copy if ignored. - if ! grep -iq $rule_name $ignorefile; then - existing_rules=$(find $output_dir/$repo_name/ -name $rule_name | wc -l) - - # For existing rules, check to see if they need to be updated, by comparing checksums - if [ $existing_rules -gt 0 ];then - local_sum=$(sha256sum $output_dir/$repo_name/$rule_name | awk '{print $1}') - if [ "$repo_sum" != "$local_sum" ]; then - echo "Checksums do not match!" - echo "Updating $rule_name..." - cp $i $output_dir/$repo_name; - ((updatecounter++)) - fi - else - # If rule doesn't exist already, we'll add it - echo "Adding new rule: $rule_name..." - cp $i $output_dir/$repo_name - ((newcounter++)) - fi - fi; -done - -# Check to see if we have any old rules that need to be removed -for i in $(find $output_dir/$repo_name -name "*.yar*" | awk -F '/' '{print $NF}'); do - is_repo_rule=$(find $clone_dir -name "$i" | wc -l) - if [ $is_repo_rule -eq 0 ]; then - echo "Could not find $i in source $repo_name repo...removing from $output_dir/$repo_name..." - rm $output_dir/$repo_name/$i - ((deletecounter++)) - fi + echo "Adding rule: $rule_name..." + cp $i $output_dir/$repo_name + ((newcounter++)) done echo "Done!" if [ "$newcounter" -gt 0 ];then - echo "$newcounter new rules added." -fi - -if [ "$updatecounter" -gt 0 ];then - echo "$updatecounter rules updated." -fi - -if [ "$deletecounter" -gt 0 ];then - echo "$deletecounter rules removed because they were deprecated or don't exist in the source repo." + echo "$newcounter rules added." fi {% else %} @@ -99,69 +56,32 @@ if [ "$gh_status" == "200" ] || [ "$gh_status" == "301" ]; then if ! $(echo "$repo" | grep -qE '^#'); then # Remove old repo if existing bc of previous error condition or unexpected disruption repo_name=`echo $repo | awk -F '/' '{print $NF}'` - [ -d $repo_name ] && rm -rf $repo_name + [ -d $output_dir/$repo_name ] && rm -rf $output_dir/$repo_name # Clone repo and make appropriate directories for rules - git clone $repo $clone_dir/$repo_name echo "Analyzing rules from $clone_dir/$repo_name..." mkdir -p $output_dir/$repo_name + # Ensure a copy of the license is available for the rules [ -f $clone_dir/$repo_name/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name # Copy over rules for i in $(find $clone_dir/$repo_name -name "*.yar*"); do rule_name=$(echo $i | awk -F '/' '{print $NF}') - repo_sum=$(sha256sum $i | awk '{print $1}') - - # Check rules against those in ignore list -- don't copy if ignored. - if ! grep -iq $rule_name $ignorefile; then - existing_rules=$(find $output_dir/$repo_name/ -name $rule_name | wc -l) - - # For existing rules, check to see if they need to be updated, by comparing checksums - if [ $existing_rules -gt 0 ];then - local_sum=$(sha256sum $output_dir/$repo_name/$rule_name | awk '{print $1}') - if [ "$repo_sum" != "$local_sum" ]; then - echo "Checksums do not match!" - echo "Updating $rule_name..." - cp $i $output_dir/$repo_name; - ((updatecounter++)) - fi - else - # If rule doesn't exist already, we'll add it - echo "Adding new rule: $rule_name..." - cp $i $output_dir/$repo_name - ((newcounter++)) - fi - fi; - done - - # Check to see if we have any old rules that need to be removed - for i in $(find $output_dir/$repo_name -name "*.yar*" | awk -F '/' '{print $NF}'); do - is_repo_rule=$(find $clone_dir/$repo_name -name "$i" | wc -l) - if [ $is_repo_rule -eq 0 ]; then - echo "Could not find $i in source $repo_name repo...removing from $output_dir/$repo_name..." - rm $output_dir/$repo_name/$i - ((deletecounter++)) - fi - done - rm -rf $clone_dir/$repo_name - fi - done < $repos + echo "Adding rule: $rule_name..." + cp $i $output_dir/$repo_name + ((newcounter++)) + done + rm -rf $clone_dir/$repo_name + fi + done < $repos echo "Done!" - + if [ "$newcounter" -gt 0 ];then - echo "$newcounter new rules added." + echo "$newcounter rules added." fi - - if [ "$updatecounter" -gt 0 ];then - echo "$updatecounter rules updated." - fi - - if [ "$deletecounter" -gt 0 ];then - echo "$deletecounter rules removed because they were deprecated or don't exist in the source repo." - fi - + else echo "Server returned $gh_status status code." echo "No connectivity to Github...exiting..." From 4d41d3aee1297f80f234eb2e3afa158484d4b1b2 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 15 Sep 2021 10:29:11 -0400 Subject: [PATCH 080/136] Ignore these rules by default because they are causing issues with YARA compilation with Strelka --- salt/strelka/defaults.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 salt/strelka/defaults.yaml diff --git a/salt/strelka/defaults.yaml b/salt/strelka/defaults.yaml new file mode 100644 index 000000000..2a3805283 --- /dev/null +++ b/salt/strelka/defaults.yaml @@ -0,0 +1,9 @@ +strelka: + ignore: + - generic_anomalies.yar + - general_cloaking.yar + - thor_inverse_matches.yar + - yara_mixed_ext_vars.yar + - gen_susp_js_obfuscatorio.yar + - apt_flame2_orchestrator.yar + - apt_tetris.yar From 39e5ded58d1037cd261e18a043ca4d933dfb8e1f Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 15 Sep 2021 11:32:29 -0400 Subject: [PATCH 081/136] Refactor ignore list and only ignore for signature-base for now --- salt/strelka/init.sls | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index d90484fed..d66b2bef2 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -21,6 +21,8 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set STRELKA_RULES = salt['pillar.get']('strelka:rules', '1') %} {% set ENGINE = salt['pillar.get']('global:mdengine', '') %} +{% import_yaml 'strelka/defaults.yaml' as strelka_config with context %} +{% set IGNORELIST = salt['pillar.get']('strelka:ignore', strelka_config.strelka.ignore, merge=True, merge_nested_lists=True) %} # Strelka config strelkaconfdir: @@ -54,6 +56,17 @@ strelkarules: - source: salt://strelka/rules - user: 939 - group: 939 + - clean: True + - exclude_pat: + {% for IGNOREDRULE in IGNORELIST %} + - {{ IGNOREDRULE }} + {% endfor %} + + {% for IGNOREDRULE in IGNORELIST %} +remove_rule_{{ IGNOREDRULE }}: + file.absent: + - name: /opt/so/conf/strelka/rules/signature-base/{{ IGNOREDRULE }} + {% endfor %} {% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone', 'so-import'] %} strelkarepos: From e3e2e1d851760c9e774b5c735d1a6b9a32a5da6f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 13:09:04 -0400 Subject: [PATCH 082/136] logic for truecluster to map file --- salt/elasticsearch/config.map.jinja | 30 +++++++++++++++++++++++++++-- salt/elasticsearch/defaults.yaml | 1 - 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index e6860e699..638747ec2 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,5 +1,31 @@ -{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG %} +# soup needs +# elasticsearch:esclustername pillar move to elasticsearch:cluster_settings:cluster:name +# move elasticsearch:true_cluster_name to elasticsearch:cluster_settings:cluster:name if true_cluster enabled +# elasticsearch:node_route_type moved + +{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} {% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} - {% do ESCONFIG.elasticsearch.defaults.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} + {% do ESCONFIG.elasticsearch.cluster_settings.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} {% endif %} + +{% if salt['pillar.get']('elasticsearch:true_cluster', False) %} + {% if grains.id.split('_') | last in ['manager','managersearch'] %} + {% if salt['pillar.get']('nodestab', {}) %} + {% set ESCONFIG.elasticsearch.cluster_settings.node.roles = ['master', 'data', 'remote_cluster_client'] %} +node.roles: [ master, data, remote_cluster_client ] + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} + {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} + {% endfor %} + {% endif %} + {% if grains.id.split('_') | last == 'managersearch' %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} + {% endif %} + {% else %} + {% set ESCONFIG.elasticsearch.cluster_settings.node.roles = ['data', 'ingest'] %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} + {% endif %} +{% endif %} + diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index bfe0e150b..817f63b0f 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,7 +1,6 @@ elasticsearch: es_port: 9200 esheap: 4049m - esclustername: default-cluster-name log_size_limit: 95 #used for curator index_settings: From aff571faf276d1856ee4cb80b6c56242410fd2a7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 15 Sep 2021 13:32:52 -0400 Subject: [PATCH 083/136] soup changes 2.3.80 --- salt/common/tools/sbin/soup | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 17339fdd5..c8935969d 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -385,6 +385,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20 [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50 + [[ "$INSTALLEDVERSION" == 2.3.50 || "$INSTALLEDVERSION" == 2.3.51 || "$INSTALLEDVERSION" == 2.3.52 || "$INSTALLEDVERSION" == 2.3.60 || "$INSTALLEDVERSION" == 2.3.61 || "$INSTALLEDVERSION" == 2.3.70]] && up_2.3.5X_to_2.3.80 true } @@ -622,6 +623,20 @@ EOF INSTALLEDVERSION=2.3.50 } +up_2.3.5X_to_2.3.80() { + + # Remove watermark settings from global.sls + sed -i '/ cluster_routing_allocation_disk/d' /opt/so/saltstack/local/pillar/global.sls + + # Add new indices to the global + sed -i '/ index_settings:/a \\ so-elasticsearch: \n shards: 1 \n warm: 7 \n close: 30 \n delete: 365' /opt/so/saltstack/local/pillar/global.sls + sed -i '/ index_settings:/a \\ so-logstash: \n shards: 1 \n warm: 7 \n close: 30 \n delete: 365' /opt/so/saltstack/local/pillar/global.sls + sed -i '/ index_settings:/a \\ so-kibana: \n shards: 1 \n warm: 7 \n close: 30 \n delete: 365' /opt/so/saltstack/local/pillar/global.sls + sed -i '/ index_settings:/a \\ so-redis: \n shards: 1 \n warm: 7 \n close: 30 \n delete: 365' /opt/so/saltstack/local/pillar/global.sls + + INSTALLEDVERSION=2.3.80 +} + verify_upgradespace() { CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//') if [ "$CURRENTSPACE" -lt "10" ]; then From 1a03853a7cbcd9c86964dc599d3e43313788bd4b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 13:38:29 -0400 Subject: [PATCH 084/136] fix extend --- salt/elasticsearch/config.map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 638747ec2..3f39a9aa3 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -12,7 +12,7 @@ {% if salt['pillar.get']('elasticsearch:true_cluster', False) %} {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} - {% set ESCONFIG.elasticsearch.cluster_settings.node.roles = ['master', 'data', 'remote_cluster_client'] %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.roles.extend(['master', 'data', 'remote_cluster_client']) %} node.roles: [ master, data, remote_cluster_client ] {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} @@ -23,7 +23,7 @@ node.roles: [ master, data, remote_cluster_client ] {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} {% endif %} {% else %} - {% set ESCONFIG.elasticsearch.cluster_settings.node.roles = ['data', 'ingest'] %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.roles.extend(['data', 'ingest']) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} {% endif %} From 1614b70853ea4a4c7820a4b16fb4639eea5475b1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 13:45:43 -0400 Subject: [PATCH 085/136] update cluster name if true cluster --- salt/elasticsearch/config.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 3f39a9aa3..2c62a0a71 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -10,10 +10,10 @@ {% endif %} {% if salt['pillar.get']('elasticsearch:true_cluster', False) %} + {% do ESCONFIG.elasticsearch.cluster_settings.cluster.update({'name': salt['pillar.get']('elasticsearch:true_cluster_name')}) %} {# this is temporary #} {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.roles.extend(['master', 'data', 'remote_cluster_client']) %} -node.roles: [ master, data, remote_cluster_client ] {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} From ac417867ed0c41c9eec7afa375ab793b201cc200 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 15 Sep 2021 14:06:08 -0400 Subject: [PATCH 086/136] so-import-evtx - final fixes --- salt/common/tools/sbin/so-import-evtx | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin/so-import-evtx index e860881f3..9e640beaa 100644 --- a/salt/common/tools/sbin/so-import-evtx +++ b/salt/common/tools/sbin/so-import-evtx @@ -43,18 +43,19 @@ function evtx2es() { docker run --rm \ -v "$EVTX:/tmp/$RUNID.evtx" \ - so-evtx2es02 \ + --entrypoint evtx2es \ + {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} \ --host {{ MANAGERIP }} --scheme https \ --index so-beats-$INDEX_DATE --pipeline import.wel \ - --login {{ES_USER}} --pwd {{ES_PW}} "/tmp/$RUNID.evtx" 1>/dev/null 2>/dev/null - + --login {{ES_USER}} --pwd {{ES_PW}} \ + "/tmp/$RUNID.evtx" 1>/dev/null 2>/dev/null docker run --rm \ -v "$EVTX:/tmp/import.evtx" \ -v "/nsm/import/evtx-end_newest:/tmp/newest" \ -v "/nsm/import/evtx-start_oldest:/tmp/oldest" \ - --entrypoint '/calculate_timestamp.sh' \ - so-evtx2es02 + --entrypoint '/evtx_calc_timestamps.sh' \ + {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} } # if no parameters supplied, display usage @@ -109,7 +110,7 @@ for EVTX in "$@"; do mkdir -p $EVTX_DIR # import evtx and write them to import ingest pipeline - echo "- importing logs with evtx2es" + echo "- importing logs to Elasticsearch..." evtx2es "${EVTX}" $HASH # compare $START to $START_OLDEST From a75238bc3fe8a654b010606657e342a05cd5227e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 15 Sep 2021 14:13:16 -0400 Subject: [PATCH 087/136] so-import-evtx - fix ingest formatting --- salt/elasticsearch/files/ingest/import.wel | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/salt/elasticsearch/files/ingest/import.wel b/salt/elasticsearch/files/ingest/import.wel index 771d47b5d..e75098f8f 100644 --- a/salt/elasticsearch/files/ingest/import.wel +++ b/salt/elasticsearch/files/ingest/import.wel @@ -4,12 +4,7 @@ { "remove": { "field": ["event.created","timestamp", "winlog.event_data.UtcTime", "event_record_id"], "ignore_failure": true } }, { "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } }, { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } }, - { - "dissect": { - "field": "log.file.name", - "pattern" : "/tmp/%{import.id}.evtx" - } -}, + { "dissect": { "field": "log.file.name", "pattern" : "/tmp/%{import.id}.evtx" } }, { "pipeline": { "name": "common" } } ] } \ No newline at end of file From c2138343169469d7b19e4d0aa144d370cada4f82 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 15:24:40 -0400 Subject: [PATCH 088/136] update the dict --- salt/elasticsearch/config.map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 2c62a0a71..3d1e7b12a 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -13,7 +13,7 @@ {% do ESCONFIG.elasticsearch.cluster_settings.cluster.update({'name': salt['pillar.get']('elasticsearch:true_cluster_name')}) %} {# this is temporary #} {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.roles.extend(['master', 'data', 'remote_cluster_client']) %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.update('roles': ['master', 'data', 'remote_cluster_client']) %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} @@ -23,7 +23,7 @@ {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} {% endif %} {% else %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.roles.extend(['data', 'ingest']) %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.update('roles': ['data', 'ingest']) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} {% endif %} From e0dc62b6e9047844bcd43e9af58630d3591aa74d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 15:43:47 -0400 Subject: [PATCH 089/136] fix dict update --- salt/elasticsearch/config.map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 3d1e7b12a..b89e04bfe 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -13,7 +13,7 @@ {% do ESCONFIG.elasticsearch.cluster_settings.cluster.update({'name': salt['pillar.get']('elasticsearch:true_cluster_name')}) %} {# this is temporary #} {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.update('roles': ['master', 'data', 'remote_cluster_client']) %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} @@ -23,7 +23,7 @@ {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} {% endif %} {% else %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.update('roles': ['data', 'ingest']) %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['data', 'ingest']}) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} {% endif %} From a43b37f234a3f2a64a4c85b410699bee6f726d46 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 15:49:18 -0400 Subject: [PATCH 090/136] fix dict update --- salt/elasticsearch/config.map.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index b89e04bfe..45ce5ed19 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -14,9 +14,9 @@ {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update.({'seed_hosts': grains.master}) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': SN.split('_')|first}) %} {% endfor %} {% endif %} {% if grains.id.split('_') | last == 'managersearch' %} @@ -25,7 +25,7 @@ {% else %} {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['data', 'ingest']}) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': grains.master}) %} {% endif %} {% endif %} From 4cf91f6c86942d5693a0a7f3a9b67a655d9ec247 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 15:51:00 -0400 Subject: [PATCH 091/136] fix dict update --- salt/elasticsearch/config.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 45ce5ed19..beb811e4b 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -14,7 +14,7 @@ {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update.({'seed_hosts': grains.master}) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': grains.master}) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': SN.split('_')|first}) %} {% endfor %} From 3bf9685df8bd621f8cafb424a6971d40cad08381 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 17:00:16 -0400 Subject: [PATCH 092/136] fix seed_hosts append --- salt/elasticsearch/config.map.jinja | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index beb811e4b..da40f578e 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -16,7 +16,7 @@ {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': grains.master}) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': SN.split('_')|first}) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} {% endfor %} {% endif %} {% if grains.id.split('_') | last == 'managersearch' %} @@ -28,4 +28,3 @@ {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': grains.master}) %} {% endif %} {% endif %} - From 782b01e76f5bf4cd2bb2e96c28022501f77f75b3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 17:07:52 -0400 Subject: [PATCH 093/136] seed_hosts to list --- salt/elasticsearch/config.map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index da40f578e..82808971a 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -14,7 +14,7 @@ {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': grains.master}) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': [grains.master]}) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} {% endfor %} @@ -25,6 +25,6 @@ {% else %} {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['data', 'ingest']}) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': grains.master}) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': [grains.master]}) %} {% endif %} {% endif %} From 82da0041a4c2dfd133c38579d80ea2afa354483c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 16 Sep 2021 07:44:15 -0400 Subject: [PATCH 094/136] Add limited roles with restricted visibility --- salt/elasticsearch/roles/limited-analyst.json | 49 +++++++++++++++++++ salt/elasticsearch/roles/limited-auditor.json | 47 ++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 salt/elasticsearch/roles/limited-analyst.json create mode 100644 salt/elasticsearch/roles/limited-auditor.json diff --git a/salt/elasticsearch/roles/limited-analyst.json b/salt/elasticsearch/roles/limited-analyst.json new file mode 100644 index 000000000..2b3797dbc --- /dev/null +++ b/salt/elasticsearch/roles/limited-analyst.json @@ -0,0 +1,49 @@ +{ + "cluster": [ + ], + "indices": [ + { + "names": [ + "so-*" + ], + "privileges": [ + "index", + "maintenance", + "monitor", + "read", + "read_cross_cluster", + "view_index_metadata" + ] + } + ], + "applications": [ + { + "application": "kibana-.kibana", + "privileges": [ + "feature_discover.read", + "feature_dashboard.read", + "feature_canvas.read", + "feature_maps.read", + "feature_ml.read", + "feature_logs.read", + "feature_visualize.read", + "feature_infrastructure.read", + "feature_apm.read", + "feature_uptime.read", + "feature_siem.read", + "feature_dev_tools.read", + "feature_advancedSettings.read", + "feature_indexPatterns.read", + "feature_savedObjectsManagement.read", + "feature_savedObjectsTagging.read", + "feature_fleet.read", + "feature_actions.read", + "feature_stackAlerts.read" + ], + "resources": [ + "*" + ] + } + ], + "run_as": [] +} \ No newline at end of file diff --git a/salt/elasticsearch/roles/limited-auditor.json b/salt/elasticsearch/roles/limited-auditor.json new file mode 100644 index 000000000..ecab5016a --- /dev/null +++ b/salt/elasticsearch/roles/limited-auditor.json @@ -0,0 +1,47 @@ +{ + "cluster": [ + ], + "indices": [ + { + "names": [ + "so-*" + ], + "privileges": [ + "read", + "read_cross_cluster", + "monitor", + "view_index_metadata" + ] + } + ], + "applications": [ + { + "application": "kibana-.kibana", + "privileges": [ + "feature_discover.read", + "feature_dashboard.read", + "feature_canvas.read", + "feature_maps.read", + "feature_ml.read", + "feature_logs.read", + "feature_visualize.read", + "feature_infrastructure.read", + "feature_apm.read", + "feature_uptime.read", + "feature_siem.read", + "feature_dev_tools.read", + "feature_advancedSettings.read", + "feature_indexPatterns.read", + "feature_savedObjectsManagement.read", + "feature_savedObjectsTagging.read", + "feature_fleet.read", + "feature_actions.read", + "feature_stackAlerts.read" + ], + "resources": [ + "*" + ] + } + ], + "run_as": [] +} \ No newline at end of file From 3fc26312e004c777454fba98e722cd5c4f4562b6 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 16 Sep 2021 08:52:31 -0400 Subject: [PATCH 095/136] Remove x-user-id header from unauthenticated proxied requests --- salt/nginx/etc/nginx.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index b85488b7f..f0308b868 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -167,6 +167,7 @@ http { proxy_pass http://{{ manager_ip }}:9822; proxy_read_timeout 90; proxy_connect_timeout 90; + proxy_set_header x-user-id ""; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -378,6 +379,7 @@ http { proxy_pass http://{{ manager_ip }}:9822/; proxy_read_timeout 90; proxy_connect_timeout 90; + proxy_set_header x-user-id ""; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; From 889d235c4577af4820cde5115f73888a0b3e037a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 16 Sep 2021 09:15:24 -0400 Subject: [PATCH 096/136] no box type more manager in true cluster --- salt/elasticsearch/config.map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 82808971a..5f29dba3b 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -19,8 +19,8 @@ {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} {% endfor %} {% endif %} - {% if grains.id.split('_') | last == 'managersearch' %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} + {% if grains.id.split('_') | last == 'manager' %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': ''}) %} {% endif %} {% else %} {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['data', 'ingest']}) %} From 5526a2bc3affd776df03afd969f7cc5b163e1664 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 16 Sep 2021 15:32:08 -0400 Subject: [PATCH 097/136] reduce defaults.yaml --- salt/elasticsearch/config.map.jinja | 22 +- salt/elasticsearch/defaults.yaml | 1102 +-------------------------- 2 files changed, 38 insertions(+), 1086 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 5f29dba3b..81ff4d157 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,30 +1,30 @@ # soup needs -# elasticsearch:esclustername pillar move to elasticsearch:cluster_settings:cluster:name -# move elasticsearch:true_cluster_name to elasticsearch:cluster_settings:cluster:name if true_cluster enabled +# elasticsearch:esclustername pillar move to elasticsearch:config:cluster:name +# move elasticsearch:true_cluster_name to elasticsearch:config:cluster:name if true_cluster enabled # elasticsearch:node_route_type moved {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} {% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} - {% do ESCONFIG.elasticsearch.cluster_settings.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} + {% do ESCONFIG.elasticsearch.config.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} {% endif %} {% if salt['pillar.get']('elasticsearch:true_cluster', False) %} - {% do ESCONFIG.elasticsearch.cluster_settings.cluster.update({'name': salt['pillar.get']('elasticsearch:true_cluster_name')}) %} {# this is temporary #} + {% do ESCONFIG.elasticsearch.config.cluster.update({'name': salt['pillar.get']('elasticsearch:true_cluster_name')}) %} {# this is temporary #} {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': [grains.master]}) %} + {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} + {% do ESCONFIG.elasticsearch.config.discovery.update({'seed_hosts': [grains.master]}) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} + {% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(SN.split('_')|first) %} {% endfor %} {% endif %} {% if grains.id.split('_') | last == 'manager' %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': ''}) %} + {% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': ''}) %} {% endif %} {% else %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['data', 'ingest']}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': [grains.master]}) %} + {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['data', 'ingest']}) %} + {% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': 'hot'}) %} + {% do ESCONFIG.elasticsearch.config.discovery.update({'seed_hosts': [grains.master]}) %} {% endif %} {% endif %} diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 817f63b0f..12d5b27fa 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,1107 +1,59 @@ elasticsearch: - es_port: 9200 - esheap: 4049m - log_size_limit: 95 #used for curator - - index_settings: - so-beats: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-firewall: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-flow: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-ids: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-import: - shards: 1 - warm: 7 - close: 73000 - delete: 73001 - so-osquery: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-ossec: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-strelka: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-syslog: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-zeek: - shards: 5 - warm: 7 - close: 45 - delete: 365 - - cluster_settings: + config: + node: + name: {{ grains.host }} + attr: + box_type: hot cluster: - max_voting_config_exclusions: 10 - auto_shrink_voting_configuration: true - election: - duration: 500ms - initial_timeout: 100ms - max_timeout: 10s - back_off_time: 100ms - strategy: supports_voting_only - no_master_block: write - persistent_tasks: - allocation: - enable: all - recheck_interval: 30s - blocks: - read_only_allow_delete: false - read_only: false - remote: - node: - attr: "" - initial_connect_timeout: 30s - #connect: true #DEPRECATION - connections_per_cluster: 3 - follower_lag: - timeout: 90000ms + name: {{ grains.host }} routing: - use_adaptive_replica_selection: true - rebalance: - enable: all allocation: - node_concurrent_incoming_recoveries: 2 - include: - _tier: "" - node_initial_primaries_recoveries: 4 - same_shard: - host: false - total_shards_per_node: "-1" - require: - _tier: "" - #shard_state: #DEPRECATION - #reroute: #DEPRECATION - #priority: NORMAL #DEPRECATION - type: balanced disk: threshold_enabled: true watermark: - flood_stage.frozen.max_headroom: 20GB - flood_stage: 98% - high: 98% low: 95% - enable_for_single_data_node: false - flood_stage.frozen: 95% - #include_relocations: true #DEPRECATION - reroute_interval: 60s - awareness: - attributes: [] - balance: - index: 0.55 - threshold: 1.0 - shard: 0.45 - enable: all - node_concurrent_outgoing_recoveries: 2 - allow_rebalance: indices_all_active - cluster_concurrent_rebalance: 2 - node_concurrent_recoveries: 2 - exclude: - _tier: "" - indices: - tombstones: - size: 500 - close: - enable: true - max_shards_per_node.frozen: 3000 - nodes: - reconnect_interval: 10s - service: - slow_master_task_logging_threshold: 10s - slow_task_logging_threshold: 30s - publish: - timeout: 30000ms - info_timeout: 10000ms - name: {{ grains.host }} # Will change if true cluster - fault_detection: - leader_check: - interval: 1000ms - timeout: 10000ms - retry_count: 3 - follower_check: - interval: 1000ms - timeout: 10000ms - retry_count: 3 - #join: #DEPRECATION - #timeout: 60000ms #DEPRECATION - max_shards_per_node: 1000 - #initial_master_nodes: [] # ERROR setting [cluster.initial_master_nodes] is not allowed when [discovery.type] is set to [single-node] - snapshot: - info: - max_concurrent_fetches: 5 - info: - update: - interval: 30s - timeout: 15s - stack: - templates: - enabled: true - logger: - level: INFO - bootstrap: - memory_lock: false - #system_call_filter: true #DEPRECATION - ctrlhandler: true - #processors: 8 #DEPRECATION - ingest: - user_agent: - cache_size: 1000 - geoip: - cache_size: 1000 - downloader: - enabled: false - endpoint: https://geoip.elastic.co/v1/database - poll: - interval: 3d - grok: - watchdog: - max_execution_time: 1s - interval: 1s + high: 98% + flood_stage: 98% network: - host: - - 0.0.0.0 - tcp: - reuse_address: true - keep_count: "-1" - #connect_timeout: 30s #DEPRECATION - keep_interval: "-1" - no_delay: true - keep_alive: true - receive_buffer_size: "-1b" - keep_idle: "-1" - send_buffer_size: "-1b" - bind_host: - - 0.0.0.0 - server: true - breaker: - inflight_requests: - limit: 100% - overhead: 2.0 - publish_host: - - 0.0.0.0 - pidfile: "" + host: 0.0.0.0 path: - data: [] logs: /var/log/elasticsearch - shared_data: "" - home: /usr/share/elasticsearch - repo: [] - search: - default_search_timeout: "-1" - highlight: - term_vector_multi_value: true - default_allow_partial_results: true - max_open_scroll_context: 500 - max_buckets: 65536 - low_level_cancellation: true - allow_expensive_queries: true - keep_alive_interval: 1m - remote: - node: - attr: "" - #initial_connect_timeout: 30s #DEPRECATION - #connect: true #DEPRECATION - #connections_per_cluster: 3 #DEPRECATION - default_keep_alive: 5m - max_keep_alive: 24h - aggs: - rewrite_to_filter_by_filter: true - security: - manager: - filter_bad_defaults: true - transform: - task_thread_pool: - queue_size: 4 - size: 4 - ccr: - wait_for_metadata_timeout: 60s - indices: - recovery: - recovery_activity_timeout: 60s - chunk_size: 1mb - internal_action_timeout: 60s - max_bytes_per_sec: 40mb - max_concurrent_file_chunks: 5 - auto_follow: - wait_for_metadata_timeout: 60s - repositories: - fs: - #compress: false #DEPRECATION - chunk_size: 9223372036854775807b - location: "" - url: - supported_protocols: - - http - - https - - ftp - - file - - jar - allowed_urls: [] - url: "http:" action: - auto_create_index: true - search: - shard_count: - limit: 9223372036854775807 destructive_requires_name: true - client: - type: node - transport: - ignore_cluster_name: false - nodes_sampler_interval: 5s - sniff: false - ping_timeout: 5s - enrich: - max_force_merge_attempts: 3 - cleanup_period: 15m - fetch_size: 10000 - coordinator_proxy: - max_concurrent_requests: 8 - max_lookups_per_request: 128 - queue_capacity: 1024 - max_concurrent_policy_executions: 50 + transport: + bind_host: 0.0.0.0 + publish_host: {{ grains.host }} + publish_port: 9300 xpack: - #flattened: #DEPRECATION - #enabled: true #DEPRECATION - watcher: - execution: - scroll: - size: 0 - timeout: "" - default_throttle_period: 5s - internal: - ops: - bulk: - default_timeout: "" - index: - default_timeout: "" - search: - default_timeout: "" - thread_pool: - queue_size: 1000 - size: 40 - index: - rest: - direct_access: "" - use_ilm_index_management: true - #history: #DEPRECATION - #cleaner_service: #DEPRECATION - #enabled: true #DEPRECATION - trigger: - schedule: - ticker: - tick_interval: 500ms - enabled: true - input: - search: - default_timeout: "" - encrypt_sensitive_data: false - transform: - search: - default_timeout: "" - stop: - timeout: 30s - watch: - scroll: - size: 0 - bulk: - concurrent_requests: 0 - flush_interval: 1s - size: 1mb - actions: 1 - actions: - bulk: - default_timeout: "" - index: - default_timeout: "" - #eql: #DEPRECATION - #enabled: true #DEPRECATION - #data_frame: #DEPRECATION - #enabled: true #DEPRECATION - #ilm: #DEPRECATION - #enabled: true #DEPRECATION - monitoring: - migration: - decommission_alerts: false - collection: - cluster: - stats: - timeout: 10s - node: - stats: - timeout: 10s - indices: [] - ccr: - stats: - timeout: 10s - enrich: - stats: - timeout: 10s - index: - stats: - timeout: 10s - recovery: - active_only: false - timeout: 10s - interval: 10s - enabled: false - ml: - job: - stats: - timeout: 10s - history: - duration: 168h - elasticsearch: - collection: - enabled: true - #enabled: true #DEPRECATION - graph: - enabled: true - searchable: - snapshot: - allocate_on_rolling_restart: false - cache: - range_size: 32mb - sync: - max_files: 10000 - interval: 60s - shutdown_timeout: 10s - recovery_range_size: 128kb - shared_cache: - recovery_range_size: 128kb - region_size: 16mb - size: 0 - min_time_delta: 60s - decay: - interval: 60s - size.max_headroom: "-1" - range_size: 16mb - max_freq: 100 - rollup: - #enabled: true #DEPRECATION - task_thread_pool: - queue_size: "-1" - size: 1 - #sql: #DEPRECATION - #enabled: true #DEPRECATION - searchable_snapshots: - cache_fetch_async_thread_pool: - core: 0 - max: 24 - keep_alive: 30s - cache_prewarming_thread_pool: - core: 0 - max: 16 - keep_alive: 30s - license: - upload: - types: - - standard - - gold - - platinum - - enterprise - - trial - self_generated: - type: basic - #logstash: #DEPRECATION - #enabled: true #DEPRECATION - notification: - pagerduty: - default_account: "" - email: - default_account: "" - html: - sanitization: - allow: - - body - - head - - _tables - - _links - - _blocks - - _formatting - - img:embedded - disallow: [] - enabled: true - reporting: - retries: 40 - warning: - enabled: true - interval: 15s - jira: - default_account: "" - slack: - default_account: "" security: - operator_privileges: - enabled: false - dls_fls: - enabled: true - dls: - bitset: - cache: - size: 10% - ttl: 2h - transport: - filter: - allow: [] - deny: [] - enabled: true - ssl: - enabled: true - verification_mode: none - certificate_authorities: /usr/share/elasticsearch/config/ca.crt - key: /usr/share/elasticsearch/config/elasticsearch.key - certificate: /usr/share/elasticsearch/config/elasticsearch.crt - ssl: - diagnose: - trust: true - enabled: true - crypto: - thread_pool: - queue_size: 1000 - size: 4 - filter: - always_allow_bound_address: true - encryption: - algorithm: AES/CTR/NoPadding - audit: - enabled: false - logfile: - emit_node_id: true - emit_node_host_name: false - emit_node_name: false - events: - emit_request_body: false - include: - - ACCESS_DENIED - - ACCESS_GRANTED - - ANONYMOUS_ACCESS_DENIED - - AUTHENTICATION_FAILED - - CONNECTION_DENIED - - TAMPERED_REQUEST - - RUN_AS_DENIED - - RUN_AS_GRANTED - - SECURITY_CONFIG_CHANGE - exclude: [] - emit_node_host_address: false authc: - password_hashing: - algorithm: bcrypt - success_cache: - size: 10000 - enabled: true - expire_after_access: 1h - api_key: - doc_cache: - ttl: 5m - cache: - hash_algo: ssha256 - max_keys: 10000 - ttl: 24h - delete: - interval: 24h - timeout: "-1" - enabled: false - hashing: - algorithm: pbkdf2 anonymous: authz_exception: true roles: [] username: _anonymous - run_as: + transport: + ssl: enabled: true - reserved_realm: - enabled: true - service_token: - cache: - hash_algo: ssha256 - max_tokens: 100000 - ttl: 20m - token: - delete: - interval: 30m - timeout: "-1" - enabled: false - thread_pool: - queue_size: 1000 - size: 1 - timeout: 20m - fips_mode: - enabled: false - encryption_key: - length: 128 - algorithm: AES + verification_mode: none + key: /usr/share/elasticsearch/config/elasticsearch.key + certificate: /usr/share/elasticsearch/config/elasticsearch.crt + certificate_authorities: + - /usr/share/elasticsearch/config/ca.crt http: - filter: - allow: [] - deny: [] - enabled: true ssl: enabled: true client_authentication: none key: /usr/share/elasticsearch/config/elasticsearch.key certificate: /usr/share/elasticsearch/config/elasticsearch.crt - certificate_authorities: /usr/share/elasticsearch/config/ca.crt - automata: - max_determinized_states: 100000 - cache: - size: 10000 - ttl: 48h - enabled: true - user: "" - authz: - store: - privileges: - cache: - ttl: 24h - max_size: 10000 - roles: - #index: #DEPRECATION - #cache: #DEPRECATION - #ttl: 20m #DEPRECATION - #max_size: 10000 #DEPRECATION - cache: - max_size: 10000 - negative_lookup_cache: - max_size: 10000 - field_permissions: - cache: - max_size_in_bytes: 104857600 - transform: - num_transform_failure_retries: 10 - #enabled: true #DEPRECATION - #vectors: #DEPRECATION - #enabled: true #DEPRECATION - ccr: - enabled: true - ccr_thread_pool: - queue_size: 100 - size: 32 - idp: - privileges: - application: "" - cache: - size: 100 - ttl: 90m - metadata: - signing: - keystore: - alias: "" - slo_endpoint: - post: "https:" - redirect: "https:" - defaults: - nameid_format: urn:oasis:names:tc:SAML:2.0:nameid-format:transient - authn_expiry: 5m - allowed_nameid_formats: - - urn:oasis:names:tc:SAML:2.0:nameid-format:transient - contact: - given_name: "" - email: "" - surname: "" - organization: - display_name: "" - name: "" - url: "http:" - sso_endpoint: - post: "https:" - redirect: "https:" - entity_id: "" - signing: - keystore: - alias: "" - sp: - cache: - size: 1000 - ttl: 60m - wildcard: - path: wildcard_services.json - enabled: false - #slm: - #enabled: true #DEPRECATION - #enrich: #DEPRECATION - #enabled: true #DEPRECATION - http: - default_connection_timeout: 10s - proxy: - host: "" - scheme: "" - port: 0 - whitelist: - - "*" - default_read_timeout: 10s - max_response_size: 10mb - autoscaling: - memory: - monitor: - timeout: 15s - ml: - max_anomaly_records: 500 - enable_config_migration: true - max_open_jobs: 512 - min_disk_space_off_heap: 5gb - use_auto_machine_memory_percent: false - inference_model: - cache_size: 40% - time_to_live: 5m - nightly_maintenance_requests_per_second: "-1.0" - node_concurrent_job_allocations: 2 - max_model_memory_limit: 0b - enabled: false - max_lazy_ml_nodes: 0 - max_ml_node_size: 0b - max_machine_memory_percent: 30 - persist_results_max_retries: 20 - autodetect_process: true - max_inference_processors: 50 - process_connect_timeout: 10s - rest: - action: - multi: - allow_explicit_index: true - cache: - recycler: - page: - limit: - heap: 10% - type: CONCURRENT - weight: - longs: 1.0 - ints: 1.0 - bytes: 1.0 - objects: 0.1 - async_search: - index_cleanup_interval: 1h - reindex: - remote: - whitelist: [] - resource: - reload: - enabled: true - interval: - low: 60s - high: 5s - medium: 30s - thread_pool: - force_merge: - queue_size: "-1" - size: 1 - fetch_shard_started: - core: 1 - max: 16 - keep_alive: 5m - #listener: #DEPRECATION - #queue_size: "-1" #DEPRECATION - #size: 4 #DEPRECATION - refresh: - core: 1 - max: 4 - keep_alive: 5m - system_write: - queue_size: 1000 - size: 4 - generic: - core: 4 - max: 128 - keep_alive: 30s - warmer: - core: 1 - max: 4 - keep_alive: 5m - search: - #max_queue_size: 1000 #DEPRECATION - queue_size: 1000 - size: 13 - #auto_queue_frame_size: 2000 #DEPRECATION - #target_response_time: 1s #DEPRECATION - #min_queue_size: 1000 #DEPRECATION - fetch_shard_store: - core: 1 - max: 16 - keep_alive: 5m - flush: - core: 1 - max: 4 - keep_alive: 5m - management: - core: 1 - max: 5 - keep_alive: 5m - analyze: - queue_size: 16 - size: 1 - get: - queue_size: 1000 - size: 8 - system_read: - queue_size: 2000 - size: 4 - estimated_time_interval: 200ms - write: - queue_size: 10000 - size: 8 - snapshot: - core: 1 - max: 4 - keep_alive: 5m - search_throttled: - #max_queue_size: 100 #DEPRECATION - queue_size: 100 - size: 1 - #auto_queue_frame_size: 200 #DEPRECATION - #target_response_time: 1s #DEPRECATION - #min_queue_size: 100 #DEPRECATION - index: - codec: default - recovery: - type: "" - store: - type: "" - fs: - fs_lock: native - preload: [] - snapshot: - uncached_chunk_size: "-1b" - cache: - excluded_file_types: [] - monitor: - jvm: - gc: - enabled: true - overhead: - warn: 50 - debug: 10 - info: 25 - refresh_interval: 1s - refresh_interval: 1s - process: - refresh_interval: 1s - os: - refresh_interval: 1s - fs: - health: - enabled: true - refresh_interval: 120s - slow_path_logging_threshold: 5s - refresh_interval: 1s - runtime_fields: - grok: - watchdog: - max_execution_time: 1s - interval: 1s - transport: - tcp: - reuse_address: true - keep_count: "-1" - #connect_timeout: 30s #DEPRECATION - keep_interval: "-1" - #compress: false #DEPRECATION - #port: 9300-9400 #DEPRECATION - no_delay: true - keep_alive: true - receive_buffer_size: "-1b" - keep_idle: "-1" - send_buffer_size: "-1b" - bind_host: - - 0.0.0.0 - connect_timeout: 30s - compress: false - ping_schedule: "-1" - connections_per_node: - recovery: 2 - state: 1 - bulk: 3 - reg: 6 - ping: 1 - tracer: - include: [] - exclude: - - internal:discovery/zen/fd* - - internal:coordination/fault_detection/* - - cluster:monitor/nodes/liveness - type: security4 - slow_operation_logging_threshold: 5s - type.default: netty4 - features: - x-pack: true - port: 9300-9400 - host: [] - publish_port: 9300 - #tcp_no_delay: true #DEPRECATION - publish_host: {{ grains.host }} - netty: - receive_predictor_size: 64kb - receive_predictor_max: 64kb - worker_count: 8 - receive_predictor_min: 64kb - boss_count: 1 + certificate_authorities: + - /usr/share/elasticsearch/config/ca.crt script: - #allowed_contexts: none # ERROR have to set to none - should be list - max_compilations_rate: 20000/1m #DEPRECATION - #cache: #DEPRECATION - #max_size: 100 #DEPRECATION - #expire: 0ms #DEPRECATION - painless: - regex: - enabled: limited - limit-factor: 6 - max_size_in_bytes: 65535 - #allowed_types: none # ERROR have to set to none - should be list - disable_max_compilations_rate: false - indexing_pressure: - memory: - limit: 10% - node: - #data: true #DEPRECATION - # roles: - # - data_frozen - # - data_warm - # - transform ERROR - # - data - # - remote_cluster_client - # - data_cold - # - data_content - # - data_hot - # - ingest - # - master - #max_local_storage_nodes: 1 #DEPRECATION - processors: 8 - store: - allow_mmap: true - #ingest: true #DEPRECATION - #master: true #DEPRECATION - pidfile: "" - #transform: true #DEPRECATION - #remote_cluster_client: true #DEPRECATION - enable_lucene_segment_infos_trace: false - #local_storage: true #DEPRECATION - name: {{ grains.host }} - id: - seed: 0 - #voting_only: false #DEPRECATION - attr: - #transform: ERROR - # node: true ERROR - xpack: - installed: "" - box_type: hot - portsfile: false - #ml: true #DEPRECATION + max_compilations_rate: 20000/1m indices: - replication: - retry_timeout: 60s - initial_retry_backoff_bound: 50ms - cache: - cleanup_interval: 1m - mapping: - dynamic_timeout: 30s - max_in_flight_updates: 10 - memory: - interval: 5s - max_index_buffer_size: "-1" - shard_inactive_time: 5m - index_buffer_size: 10% - min_index_buffer_size: 48mb - breaker: - request: - limit: 60% - type: memory - overhead: 1.0 - total: - limit: 95% - use_real_memory: true - accounting: - limit: 100% - overhead: 1.0 - fielddata: - limit: 40% - type: memory - overhead: 1.03 - type: hierarchy query: bool: - max_nested_depth: 20 max_clause_count: 1500 - query_string: - analyze_wildcard: false - allowLeadingWildcard: true - id_field_data: - enabled: true - recovery: - recovery_activity_timeout: 1800000ms - retry_delay_network: 5s - internal_action_timeout: 15m - retry_delay_state_sync: 500ms - internal_action_long_timeout: 1800000ms - max_concurrent_operations: 1 - max_bytes_per_sec: 40mb - max_concurrent_file_chunks: 2 - requests: - cache: - size: 1% - #expire: 1ms #0ms - ERROR when set to 0ms, set to 1ms and ERROR gone - store: - delete: - shard: - timeout: 30s - analysis: - hunspell: - dictionary: - ignore_case: false - lazy: false - queries: - cache: - count: 10000 - size: 10% - all_segments: false - lifecycle: - history_index_enabled: true - poll_interval: 10m - step: - master_timeout: 30s - fielddata: - cache: - size: "-1b" - plugin: - mandatory: [] - slm: - minimum_interval: 15m - retention_schedule: 0 30 1 * * ? - retention_duration: 1h - history_index_enabled: true - discovery: - #seed_hosts: [] # ERROR - it is forbidden to set both [discovery.seed_hosts] and [discovery.zen.ping.unicast.hosts] - unconfigured_bootstrap_timeout: 3s - request_peers_timeout: 3000ms - zen: - #commit_timeout: 30s #DEPRECATION - #no_master_block: write #DEPRECATION - #join_retry_delay: 100ms #DEPRECATION - #join_retry_attempts: 3 #DEPRECATION - #ping: - #unicast: - #concurrent_connects: 10 # ERROR forbidden to set both [discovery.seed_resolver.max_concurrent_resolvers] and [discovery.zen.ping.unicast.concurrent_connects] - #hosts: [] # ERROR - it is forbidden to set both [discovery.seed_hosts] and [discovery.zen.ping.unicast.hosts] - #hosts.resolve_timeout: 5s # ERROR forbidden to set both [discovery.seed_resolver.timeout] and [discovery.zen.ping.unicast.hosts.resolve_timeout] - #master_election: #DEPRECATION - #ignore_non_master_pings: false #DEPRECATION - #wait_for_joins_timeout: 30000ms #DEPRECATION - #send_leave_request: true #DEPRECATION - ping_timeout: 3s - #bwc_ping_timeout: 3s #DEPRECATION - #join_timeout: 60000ms #DEPRECATION - #publish_diff: - #enable: true #DEPRECATION - #publish: #DEPRECATION - #max_pending_cluster_states: 25 #DEPRECATION - #minimum_master_nodes: "-1" #DEPRECATION - #unsafe_rolling_upgrades_enabled: true #DEPRECATION - #hosts_provider: # ERROR forbidden to set both [discovery.seed_providers] and [discovery.zen.hosts_provider] has to be commented out - #publish_timeout: 30s #DEPRECATION - #fd: #DEPRECATION - #connect_on_network_disconnect: false #DEPRECATION - #ping_interval: 1s #DEPRECATION - #ping_retries: 3 #DEPRECATION - #register_connection_listener: true #DEPRECATION - #ping_timeout: 30s #DEPRECATION - #max_pings_from_another_master: 3 #DEPRECATION - initial_state_timeout: 30s - cluster_formation_warning_timeout: 10000ms - #seed_providers: # ERROR forbidden to set both [discovery.seed_providers] and [discovery.zen.hosts_provider] has to be commented out - type: zen # ERROR java.lang.IllegalArgumentException: node with [discovery.type] set to [single-node] must be master-eligible # test turning off - #seed_resolver: - #max_concurrent_resolvers: 10 # ERROR forbidden to set both [discovery.seed_resolver.max_concurrent_resolvers] and [discovery.zen.ping.unicast.concurrent_connects] - #timeout: 5s # forbidden to set both [discovery.seed_resolver.timeout] and [discovery.zen.ping.unicast.hosts.resolve_timeout] - find_peers_interval: 1000ms - probe: - connect_timeout: 30s - handshake_timeout: 30s - http: - cors: - max-age: 1728000 - allow-origin: "" - allow-headers: X-Requested-With,Content-Type,Content-Length - allow-credentials: false - allow-methods: OPTIONS,HEAD,GET,POST,PUT,DELETE - enabled: false - max_chunk_size: 8kb - compression_level: 3 - max_initial_line_length: 4kb - type: security4 - pipelining: - max_events: 10000 - type.default: netty4 - #content_type: #DEPRECATION - #required: true #DEPRECATION - host: [] - publish_port: "-1" - read_timeout: 0ms - max_content_length: 100mb - netty: - receive_predictor_size: 64kb - max_composite_buffer_components: 69905 - worker_count: 0 - tcp: - reuse_address: true - keep_count: "-1" - keep_interval: "-1" - no_delay: true - keep_alive: true - receive_buffer_size: "-1b" - keep_idle: "-1" - send_buffer_size: "-1b" - bind_host: [] - client_stats: - enabled: true - reset_cookies: false - max_warning_header_count: "-1" - tracer: - include: [] - exclude: [] - max_warning_header_size: "-1b" - detailed_errors: - enabled: true - port: 9200-9300 - max_header_size: 8kb - #tcp_no_delay: true #DEPRECATION - compression: false - publish_host: [] - gateway: - #recover_after_master_nodes: 0 #DEPRECATION - #expected_nodes: "-1" #DEPRECATION - recover_after_data_nodes: "-1" - expected_data_nodes: "-1" - write_dangling_indices_info: true - slow_write_logging_threshold: 10s - recover_after_time: 0ms - #expected_master_nodes: "-1" #DEPRECATION - #recover_after_nodes: "-1" #DEPRECATION - #auto_import_dangling_indices: false #DEPRECATION - snapshot: - refresh_repo_uuid_on_restore: true - max_concurrent_operations: 1000 + + + + From fbd9bab2f14f6924e49fcdc1f9ab0205047c7bcc Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 16 Sep 2021 16:08:55 -0400 Subject: [PATCH 098/136] Split apart roles and users into separate maps --- salt/soc/files/soc/soc.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index e9dfa50d8..e33ea406b 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -91,8 +91,10 @@ "roleFiles": [ "rbac/permissions", "rbac/roles", - "rbac/users_roles", "rbac/custom_roles" + ], + "userFiles": [ + "rbac/users_roles" ] } }, From 3ce48acadd1b6e3711e72882c816a0ba723a24bf Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 16 Sep 2021 16:44:31 -0400 Subject: [PATCH 099/136] change cluster_settings to config --- salt/elasticsearch/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 3cb58f71b..f5d983869 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -152,7 +152,7 @@ esyml: - user: 930 - group: 939 - defaults: - ESCONFIG: {{ ESCONFIG.elasticsearch.cluster_settings }} + ESCONFIG: {{ ESCONFIG.elasticsearch.config }} - template: jinja From 0d074dafd4583281d3cf4f2a01fa576f1b723954 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 17 Sep 2021 09:52:50 -0400 Subject: [PATCH 100/136] add missing defaults --- salt/elasticsearch/defaults.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 12d5b27fa..e119fb62d 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -25,7 +25,10 @@ elasticsearch: publish_host: {{ grains.host }} publish_port: 9300 xpack: + ml: + enabled: false security: + enabled: true authc: anonymous: authz_exception: true From d0c87cd31748e2a3b44cd4f27bca07b2ebcc97fb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 17 Sep 2021 12:11:12 -0400 Subject: [PATCH 101/136] allow for pillar override of defaults --- salt/elasticsearch/config.map.jinja | 2 ++ salt/elasticsearch/init.sls | 10 +--------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 81ff4d157..ec05c0e14 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -28,3 +28,5 @@ {% do ESCONFIG.elasticsearch.config.discovery.update({'seed_hosts': [grains.master]}) %} {% endif %} {% endif %} + +{% set ESCONFIG = salt['pillar.get']('elasticsearch:config', default=ESCONFIG.elasticsearch.config, merge=True) %} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index f5d983869..9f475c2c3 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -152,17 +152,9 @@ esyml: - user: 930 - group: 939 - defaults: - ESCONFIG: {{ ESCONFIG.elasticsearch.config }} + ESCONFIG: {{ ESCONFIG }} - template: jinja - -esyml_test: - file.managed: - - name: /tmp/elasticsearch.yml - - user: 930 - - group: 939 - - contents: {{ ESCONFIG | yaml }} - #sync templates to /opt/so/conf/elasticsearch/templates {% for TEMPLATE in TEMPLATES %} es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: From 4d307c53e85dac5cec96a52bc90c45964d510090 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 17 Sep 2021 13:01:50 -0400 Subject: [PATCH 102/136] Add support for disabling Zeek and Suricata --- salt/pcap/init.sls | 3 ++- salt/suricata/init.sls | 22 +++++++++++++++++----- salt/suricata/map.jinja | 17 +++++++++++++---- salt/zeek/init.sls | 35 +++++++++++++++++++++++++++++------ salt/zeek/map.jinja | 17 +++++++++++++---- 5 files changed, 74 insertions(+), 20 deletions(-) diff --git a/salt/pcap/init.sls b/salt/pcap/init.sls index a64b9d90a..641300fdf 100644 --- a/salt/pcap/init.sls +++ b/salt/pcap/init.sls @@ -15,13 +15,14 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} +{% from "pcap/map.jinja" import STENOOPTIONS with context %} + {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set INTERFACE = salt['pillar.get']('sensor:interface', 'bond0') %} {% set BPF_STENO = salt['pillar.get']('steno:bpf', None) %} {% set BPF_COMPILED = "" %} -{% from "pcap/map.jinja" import STENOOPTIONS with context %} # PCAP Section diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index 4cbda4bb4..73c4d2395 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -15,6 +15,8 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states and grains.role not in ['so-manager', 'so-managersearch'] %} +{% from "suricata/map.jinja" import SURICATAOPTIONS with context %} + {% set interface = salt['pillar.get']('sensor:interface', 'bond0') %} {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} @@ -136,9 +138,10 @@ suribpf: {% endif %} so-suricata: - docker_container.running: + docker_container.{{ SURICATAOPTIONS.status }}: + {% if SURICATAOPTIONS.status == 'running' %} - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-suricata:{{ VERSION }} - - start: {{ START }} + - start: {{ SURICATAOPTIONS.start }} - privileged: True - environment: - INTERFACE={{ interface }} @@ -157,18 +160,27 @@ so-suricata: - file: /opt/so/conf/suricata/rules/ - file: /opt/so/conf/suricata/bpf + {% else %} {# if Suricata isn't enabled, then stop and remove the container #} + - force: True + {% endif %} + append_so-suricata_so-status.conf: file.append: - name: /opt/so/conf/so-status/so-status.conf - text: so-suricata - unless: grep -q so-suricata /opt/so/conf/so-status/so-status.conf -{% if grains.role == 'so-import' %} -disable_so-suricata_so-status.conf: + {% if not SURICATAOPTIONS.start %} +so-suricata_so-status.disabled: file.comment: - name: /opt/so/conf/so-status/so-status.conf - regex: ^so-suricata$ -{% endif %} + {% else %} +delete_so-suricata_so-status.disabled: + file.uncomment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-suricata$ + {% endif %} /usr/local/bin/surirotate: cron.absent: diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index ad4d70e80..2ffd5ad08 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -1,6 +1,15 @@ -# don't start the docker container if it is an import node -{% if grains.id.split('_')|last == 'import' %} - {% set START = False %} +{% set SURICATAOPTIONS = {} %} +{% set ENABLED = salt['pillar.get']('suricata:enabled', 'True') %} + +# don't start the docker container if it is an import node or disabled via pillar +{% if grains.id.split('_')|last == 'import' or ENABLED is sameas false %} + {% do SURICATAOPTIONS.update({'start': False}) %} {% else %} - {% set START = True %} + {% do SURICATAOPTIONS.update({'start': True}) %} +{% endif %} + +{% if ENABLED is sameas false %} + {% do SURICATAOPTIONS.update({'status': 'absent'}) %} +{% else %} + {% do SURICATAOPTIONS.update({'status': 'running'}) %} {% endif %} \ No newline at end of file diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index ce0d6dccd..5f5adfaa6 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -1,7 +1,21 @@ +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -{% from "zeek/map.jinja" import START with context %} +{% from "zeek/map.jinja" import ZEEKOPTIONS with context %} {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} @@ -179,9 +193,10 @@ localzeeksync: LOCAL: {{ ZEEK.local | tojson }} so-zeek: - docker_container.running: + docker_container.{{ ZEEKOPTIONS.status }}: + {% if ZEEKOPTIONS.status == 'running' %} - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }} - - start: {{ START }} + - start: {{ ZEEKOPTIONS.start }} - privileged: True - ulimits: - core=0 @@ -204,6 +219,9 @@ so-zeek: - file: /opt/so/conf/zeek/zeekctl.cfg - file: /opt/so/conf/zeek/policy - file: /opt/so/conf/zeek/bpf + {% else %} {# if Zeek isn't enabled, then stop and remove the container #} + - force: True + {% endif %} append_so-zeek_so-status.conf: file.append: @@ -211,12 +229,17 @@ append_so-zeek_so-status.conf: - text: so-zeek - unless: grep -q so-zeek /opt/so/conf/so-status/so-status.conf -{% if grains.role == 'so-import' %} -disable_so-zeek_so-status.conf: + {% if not ZEEKOPTIONS.start %} +so-zeek_so-status.disabled: file.comment: - name: /opt/so/conf/so-status/so-status.conf - regex: ^so-zeek$ -{% endif %} + {% else %} +delete_so-zeek_so-status.disabled: + file.uncomment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-zeek$ + {% endif %} {% else %} diff --git a/salt/zeek/map.jinja b/salt/zeek/map.jinja index ad4d70e80..b5713c6d5 100644 --- a/salt/zeek/map.jinja +++ b/salt/zeek/map.jinja @@ -1,6 +1,15 @@ -# don't start the docker container if it is an import node -{% if grains.id.split('_')|last == 'import' %} - {% set START = False %} +{% set ZEEKOPTIONS = {} %} +{% set ENABLED = salt['pillar.get']('zeek:enabled', 'True') %} + +# don't start the docker container if it is an import node or disabled via pillar +{% if grains.id.split('_')|last == 'import' or ENABLED is sameas false %} + {% do ZEEKOPTIONS.update({'start': False}) %} {% else %} - {% set START = True %} + {% do ZEEKOPTIONS.update({'start': True}) %} +{% endif %} + +{% if ENABLED is sameas false %} + {% do ZEEKOPTIONS.update({'status': 'absent'}) %} +{% else %} + {% do ZEEKOPTIONS.update({'status': 'running'}) %} {% endif %} \ No newline at end of file From b877aa44bce0fbdb14a03fcfbb26c85d33002199 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 17 Sep 2021 14:10:45 -0400 Subject: [PATCH 103/136] update dict --- salt/elasticsearch/config.map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index ec05c0e14..a6a6ec9c5 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -14,7 +14,7 @@ {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} - {% do ESCONFIG.elasticsearch.config.discovery.update({'seed_hosts': [grains.master]}) %} + {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} {% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(SN.split('_')|first) %} {% endfor %} @@ -25,7 +25,7 @@ {% else %} {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['data', 'ingest']}) %} {% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': 'hot'}) %} - {% do ESCONFIG.elasticsearch.config.discovery.update({'seed_hosts': [grains.master]}) %} + {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %} {% endif %} {% endif %} From e864bc54044362007b45f88562a06d385eaeb852 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 17 Sep 2021 15:28:35 -0400 Subject: [PATCH 104/136] move custom es cluster name pillar location --- setup/so-functions | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d851c80b1..fbc06fd84 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1725,16 +1725,17 @@ manager_global() { "elastic:"\ " features: False"\ "elasticsearch:"\ - " replicas: 0" >> "$global_pillar" if [ -n "$ESCLUSTERNAME" ]; then printf '%s\n'\ " true_cluster: True"\ - " true_cluster_name: '$ESCLUSTERNAME'" >> "$global_pillar" + " config:"\ + " cluster:"\ + " name: '$ESCLUSTERNAME'" >> "$global_pillar" else printf '%s\n'\ - " true_cluster: False"\ - " true_cluster_name: 'so'" >> "$global_pillar" + " true_cluster: False" >> "$global_pillar" fi + " replicas: 0" >> "$global_pillar" printf '%s\n'\ " discovery_nodes: 1"\ " hot_warm_enabled: False"\ From 109c83d8c3b84cb79e02e312272159828ce5b557 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 17 Sep 2021 15:29:41 -0400 Subject: [PATCH 105/136] move custom es cluster name pillar location --- salt/elasticsearch/config.map.jinja | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index a6a6ec9c5..d0944842c 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -10,8 +10,7 @@ {% endif %} {% if salt['pillar.get']('elasticsearch:true_cluster', False) %} - {% do ESCONFIG.elasticsearch.config.cluster.update({'name': salt['pillar.get']('elasticsearch:true_cluster_name')}) %} {# this is temporary #} - {% if grains.id.split('_') | last in ['manager','managersearch'] %} + {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %} From 01323cc192721e35316b67eb55e8ab1ea8c9753b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 17 Sep 2021 15:44:54 -0400 Subject: [PATCH 106/136] fix clustername redirect --- setup/so-functions | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index fbc06fd84..06ec7238a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1724,7 +1724,7 @@ manager_global() { " hot_warm: False"\ "elastic:"\ " features: False"\ - "elasticsearch:"\ + "elasticsearch:"\ >> "$global_pillar" if [ -n "$ESCLUSTERNAME" ]; then printf '%s\n'\ " true_cluster: True"\ @@ -1735,8 +1735,9 @@ manager_global() { printf '%s\n'\ " true_cluster: False" >> "$global_pillar" fi - " replicas: 0" >> "$global_pillar" + printf '%s\n'\ + " replicas: 0"\ " discovery_nodes: 1"\ " hot_warm_enabled: False"\ " cluster_routing_allocation_disk.threshold_enabled: true"\ From 30e781d0763c15e3d4b590d85cdefa4ea09961f3 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 17 Sep 2021 17:54:38 -0400 Subject: [PATCH 107/136] Use user ID instead of email as role master --- salt/common/tools/sbin/so-user | 103 +++++++++++++++++++++++---------- salt/soc/init.sls | 2 +- 2 files changed, 74 insertions(+), 31 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 7d4c55453..40416860e 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -49,8 +49,11 @@ databasePath=${KRATOS_DB_PATH:-/opt/so/conf/kratos/db/db.sqlite} bcryptRounds=${BCRYPT_ROUNDS:-12} elasticUsersFile=${ELASTIC_USERS_FILE:-/opt/so/saltstack/local/salt/elasticsearch/files/users} elasticRolesFile=${ELASTIC_ROLES_FILE:-/opt/so/saltstack/local/salt/elasticsearch/files/users_roles} +socRolesFile=${SOC_ROLES_FILE:-/opt/so/conf/soc/soc_users_roles} esUID=${ELASTIC_UID:-930} esGID=${ELASTIC_GID:-930} +soUID=${SOCORE_UID:-939} +soGID=${SOCORE_GID:-939} function lock() { # Obtain file descriptor lock @@ -87,7 +90,7 @@ function findIdByEmail() { email=$1 response=$(curl -Ss -L ${kratosUrl}/identities) - identityId=$(echo "${response}" | jq ".[] | select(.verifiable_addresses[0].value == \"$email\") | .id") + identityId=$(echo "${response}" | jq -r ".[] | select(.verifiable_addresses[0].value == \"$email\") | .id") echo $identityId } @@ -139,32 +142,39 @@ function updatePassword() { # Generate password hash passwordHash=$(hashPassword "$password") # Update DB with new hash - echo "update identity_credentials set config=CAST('{\"hashed_password\":\"$passwordHash\"}' as BLOB) where identity_id=${identityId};" | sqlite3 "$databasePath" + echo "update identity_credentials set config=CAST('{\"hashed_password\":\"$passwordHash\"}' as BLOB) where identity_id='${identityId}';" | sqlite3 "$databasePath" [[ $? != 0 ]] && fail "Unable to update password" fi } -function createElasticFile() { +function createFile() { filename=$1 + uid=$2 + gid=$3 + truncate -s 0 "$filename" chmod 600 "$filename" - chown "${esUID}:${esGID}" "$filename" + chown "${uid}:${gid}" "$filename" } function ensureRoleFileExists() { - if [ ! -f "$elasticRolesFile" ]; then - echo "Creating new roles file: $elasticRolesFile" - rolesTmpFile="${elasticRolesFile}.tmp" - createElasticFile "${rolesTmpFile}" - authPillarJson=$(lookup_salt_value "auth" "elasticsearch" "pillar" "json") - syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_kibana_user" "superuser" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_beats_user" "superuser" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "monitoring_user" "$rolesTmpFile" - mv "${rolesTmpFile}" "${elasticRolesFile}" + if [ ! -s "$socRolesFile" ]; then + echo "Migrating roles to new file: $socRolesFile" + + rolesTmpFile="${socRolesFile}.tmp" + createFile "$rolesTmpFile" "$soUID" "$soGID" + + if [[ -f "$databasePath" ]]; then + # Generate the new users file + echo "select 'superuser:' || id from identities;" | sqlite3 "$databasePath" \ + >> "$rolesTmpFile" + [[ $? != 0 ]] && fail "Unable to read identities from database" + else + echo "Database file does not exist yet, installation is likely not yet complete." + exit 1 + fi + + mv "${rolesTmpFile}" "${socRolesFile}" fi } @@ -196,11 +206,12 @@ function syncElasticSystemRole() { } function syncElastic() { - echo "Syncing users between SOC and Elastic..." - ensureRoleFileExists + echo "Syncing users and roles between SOC and Elastic..." usersTmpFile="${elasticUsersFile}.tmp" - createElasticFile "${usersTmpFile}" + createFile "${usersTmpFile}" "$esUID" "$esGID" + rolesTmpFile="${elasticRolesFile}.tmp" + createFile "${rolesTmpFile}" "$esUID" "$esGID" authPillarJson=$(lookup_salt_value "auth" "elasticsearch" "pillar" "json") @@ -210,8 +221,16 @@ function syncElastic() { syncElasticSystemUser "$authPillarJson" "so_beats_user" "$usersTmpFile" syncElasticSystemUser "$authPillarJson" "so_monitor_user" "$usersTmpFile" - if [[ -f "$databasePath" ]]; then - # Generate the new users file + syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_kibana_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_beats_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "monitoring_user" "$rolesTmpFile" + + if [[ -f "$databasePath" && -f "$socRolesFile" ]]; then + # Append the SOC users echo "select '{\"user\":\"' || ici.identifier || '\", \"data\":' || ic.config || '}'" \ "from identity_credential_identifiers ici, identity_credentials ic " \ "where ici.identity_credential_id=ic.id and instr(ic.config, 'hashed_password') " \ @@ -220,12 +239,24 @@ function syncElastic() { jq -r '.user + ":" + .data.hashed_password' \ >> "$usersTmpFile" [[ $? != 0 ]] && fail "Unable to read credential hashes from database" + + # Append the user roles + while IFS="" read -r rolePair || [ -n "$rolePair" ]; do + userId=$(echo "$rolePair" | cut -d: -f2) + role=$(echo "$rolePair" | cut -d: -f1) + echo "select '$role:' || ici.identifier " \ + "from identity_credential_identifiers ici, identity_credentials ic " \ + "where ici.identity_credential_id=ic.id and ic.identity_id = '$userId';" | \ + sqlite3 "$databasePath" >> "$rolesTmpFile" + done < "$socRolesFile" + else - echo "Database file does not exist yet, skipping users export" + echo "Database file or soc roles file does not exist yet, skipping users export" fi if [[ -s "${usersTmpFile}" ]]; then mv "${usersTmpFile}" "${elasticUsersFile}" + mv "${rolesTmpFile}" "${elasticRolesFile}" if [[ -z "$SKIP_STATE_APPLY" ]]; then echo "Elastic state will be re-applied to affected minions. This may take several minutes..." @@ -238,15 +269,22 @@ function syncElastic() { } function syncAll() { + ensureRoleFileExists + + # Check if a sync is needed. Sync is not needed if the following are true: + # - user database entries are all older than the elastic users file + # - soc roles file last modify date is older than the elastic roles file if [[ -z "$FORCE_SYNC" && -f "$databasePath" && -f "$elasticUsersFile" ]]; then usersFileAgeSecs=$(echo $(($(date +%s) - $(date +%s -r "$elasticUsersFile")))) staleCount=$(echo "select count(*) from identity_credentials where updated_at >= Datetime('now', '-${usersFileAgeSecs} seconds');" \ | sqlite3 "$databasePath") - if [[ "$staleCount" == "0" ]]; then + if [[ "$staleCount" == "0" && "$elasticRolesFile" -nt "$socRolesFile" ]]; then return 1 fi fi + syncElastic + return 0 } @@ -285,20 +323,20 @@ function adjustUserRole() { ensureRoleFileExists - filename="$elasticRolesFile" + filename="$socRolesFile" hasRole=0 - grep "$role:" "$elasticRolesFile" | grep -q "$email" && hasRole=1 + grep "$role:" "$socRolesFile" | grep -q "$identityId" && hasRole=1 if [[ "$op" == "add" ]]; then if [[ "$hasRole" == "1" ]]; then fail "User '$email' already has the role: $role" else - echo "$role:$email" >> "$filename" + echo "$role:$identityId" >> "$filename" fi elif [[ "$op" == "del" ]]; then if [[ "$hasRole" -ne 1 ]]; then fail "User '$email' does not have the role: $role" else - sed -i "/^$role:$email\$/d" "$filename" + sed -i "/^$role:$identityId\$/d" "$filename" fi else fail "Unsupported role adjustment operation: $op" @@ -321,7 +359,7 @@ EOF response=$(curl -Ss -L ${kratosUrl}/identities -d "$addUserJson") [[ $? != 0 ]] && fail "Unable to communicate with Kratos" - identityId=$(echo "${response}" | jq ".id") + identityId=$(echo "${response}" | jq -r ".id") if [[ ${identityId} == "null" ]]; then code=$(echo "${response}" | jq ".error.code") [[ "${code}" == "409" ]] && fail "User already exists" @@ -332,7 +370,7 @@ EOF addUserRole "$email" "$role" fi - updatePassword $identityId + updatePassword "$identityId" } function updateStatus() { @@ -382,6 +420,11 @@ function deleteUser() { response=$(curl -Ss -XDELETE -L "${kratosUrl}/identities/$identityId") [[ $? != 0 ]] && fail "Unable to communicate with Kratos" + + rolesTmpFile="${socRolesFile}.tmp" + createFile "$rolesTmpFile" "$soUID" "$soGID" + grep -v "$id" "$socRolesFile" > "$rolesTmpFile" + mv "$rolesTmpFile" "$socRolesFile" } case "${operation}" in diff --git a/salt/soc/init.sls b/salt/soc/init.sls index c3c466849..69cc54c82 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -91,7 +91,7 @@ so-soc: - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro - - /opt/so/conf/elasticsearch/users_roles:/opt/sensoroni/rbac/users_roles:ro + - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw - /opt/so/log/soc/:/opt/sensoroni/logs/:rw {%- if salt['pillar.get']('nodestab', {}) %} - extra_hosts: From 7f2d7eb0388fe4063628710364a3bdaf62abe178 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 18 Sep 2021 07:20:34 -0400 Subject: [PATCH 108/136] Continue migration of user emails to IDs --- salt/common/tools/sbin/so-user | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 40416860e..5fc1db66a 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -159,22 +159,22 @@ function createFile() { function ensureRoleFileExists() { if [ ! -s "$socRolesFile" ]; then - echo "Migrating roles to new file: $socRolesFile" - - rolesTmpFile="${socRolesFile}.tmp" - createFile "$rolesTmpFile" "$soUID" "$soGID" - if [[ -f "$databasePath" ]]; then + echo "Migrating roles to new file: $socRolesFile" + # Generate the new users file + rolesTmpFile="${socRolesFile}.tmp" + createFile "$rolesTmpFile" "$soUID" "$soGID" + echo "select 'superuser:' || id from identities;" | sqlite3 "$databasePath" \ >> "$rolesTmpFile" [[ $? != 0 ]] && fail "Unable to read identities from database" + + mv "${rolesTmpFile}" "${socRolesFile}" else echo "Database file does not exist yet, installation is likely not yet complete." - exit 1 + exit 0 fi - - mv "${rolesTmpFile}" "${socRolesFile}" fi } From 5377a1a85ed917adbe4431ca3abed49d79cb50b7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 18 Sep 2021 11:06:54 -0400 Subject: [PATCH 109/136] Recover from situation where roles file is corrupted --- salt/common/tools/sbin/so-user | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 5fc1db66a..180092333 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -158,7 +158,7 @@ function createFile() { } function ensureRoleFileExists() { - if [ ! -s "$socRolesFile" ]; then + if [[ ! -f "$socRolesFile" || ! -s "$socRolesFile" ]]; then if [[ -f "$databasePath" ]]; then echo "Migrating roles to new file: $socRolesFile" @@ -170,6 +170,7 @@ function ensureRoleFileExists() { >> "$rolesTmpFile" [[ $? != 0 ]] && fail "Unable to read identities from database" + rm -fr "$socRolesFile" mv "${rolesTmpFile}" "${socRolesFile}" else echo "Database file does not exist yet, installation is likely not yet complete." From 357c1db4451aa591bb82e320e1fe3096c0dfc2cc Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 18 Sep 2021 11:08:35 -0400 Subject: [PATCH 110/136] Recover from situation where roles file is corrupted --- salt/common/tools/sbin/so-user | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 180092333..61aee71d6 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -172,6 +172,8 @@ function ensureRoleFileExists() { rm -fr "$socRolesFile" mv "${rolesTmpFile}" "${socRolesFile}" + echo "The following users have all been migrated with the super user role:" + cat "${socRolesFile}" else echo "Database file does not exist yet, installation is likely not yet complete." exit 0 From 5704906b110b9132be7081bb20707c4ed124f4c1 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 18 Sep 2021 15:49:05 -0400 Subject: [PATCH 111/136] Create empty files for Docker to mount while installation continues --- salt/common/tools/sbin/so-user | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 61aee71d6..0af5eab16 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -170,12 +170,16 @@ function ensureRoleFileExists() { >> "$rolesTmpFile" [[ $? != 0 ]] && fail "Unable to read identities from database" - rm -fr "$socRolesFile" mv "${rolesTmpFile}" "${socRolesFile}" echo "The following users have all been migrated with the super user role:" cat "${socRolesFile}" else echo "Database file does not exist yet, installation is likely not yet complete." + + # Create all required files to allow docker volumes to bind correctly + createFile "$socRolesFile" "$soUID" "$soGID" + createFile "$elasticUsersFile" "$esUID" "$esGID" + createFile "$elasticRolesFile" "$esUID" "$esGID" exit 0 fi fi From 3508f3d8c18f5152c435451b6ee29b07d5e5f93f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 18 Sep 2021 19:20:43 -0400 Subject: [PATCH 112/136] Ensure ES user/role files are generated even if the primary admin user isn't yet created, since the system users are necessary for other installation functions --- salt/common/tools/sbin/so-user | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 0af5eab16..269f1867b 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -152,6 +152,7 @@ function createFile() { uid=$2 gid=$3 + mkdir -p $(dirname "$filename") truncate -s 0 "$filename" chmod 600 "$filename" chown "${uid}:${gid}" "$filename" @@ -159,29 +160,24 @@ function createFile() { function ensureRoleFileExists() { if [[ ! -f "$socRolesFile" || ! -s "$socRolesFile" ]]; then + # Generate the new users file + rolesTmpFile="${socRolesFile}.tmp" + createFile "$rolesTmpFile" "$soUID" "$soGID" + if [[ -f "$databasePath" ]]; then echo "Migrating roles to new file: $socRolesFile" - # Generate the new users file - rolesTmpFile="${socRolesFile}.tmp" - createFile "$rolesTmpFile" "$soUID" "$soGID" - echo "select 'superuser:' || id from identities;" | sqlite3 "$databasePath" \ >> "$rolesTmpFile" [[ $? != 0 ]] && fail "Unable to read identities from database" - mv "${rolesTmpFile}" "${socRolesFile}" echo "The following users have all been migrated with the super user role:" cat "${socRolesFile}" else echo "Database file does not exist yet, installation is likely not yet complete." - - # Create all required files to allow docker volumes to bind correctly - createFile "$socRolesFile" "$soUID" "$soGID" - createFile "$elasticUsersFile" "$esUID" "$esGID" - createFile "$elasticRolesFile" "$esUID" "$esGID" - exit 0 fi + + mv "${rolesTmpFile}" "${socRolesFile}" fi } From 730503b69c7fe5069803d0bd304b7376ce35ecd9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 18 Sep 2021 23:17:49 -0400 Subject: [PATCH 113/136] Ensure highstate migrates user roles --- salt/common/tools/sbin/so-user | 22 +++++++++++++--------- salt/manager/init.sls | 1 + 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 269f1867b..df77ed1e6 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -138,7 +138,7 @@ function updatePassword() { validatePassword "$password" fi - if [[ -n $identityId ]]; then + if [[ -n "$identityId" ]]; then # Generate password hash passwordHash=$(hashPassword "$password") # Update DB with new hash @@ -172,7 +172,7 @@ function ensureRoleFileExists() { [[ $? != 0 ]] && fail "Unable to read identities from database" echo "The following users have all been migrated with the super user role:" - cat "${socRolesFile}" + cat "${rolesTmpFile}" else echo "Database file does not exist yet, installation is likely not yet complete." fi @@ -331,7 +331,8 @@ function adjustUserRole() { grep "$role:" "$socRolesFile" | grep -q "$identityId" && hasRole=1 if [[ "$op" == "add" ]]; then if [[ "$hasRole" == "1" ]]; then - fail "User '$email' already has the role: $role" + echo "User '$email' already has the role: $role" + return 1 else echo "$role:$identityId" >> "$filename" fi @@ -339,11 +340,14 @@ function adjustUserRole() { if [[ "$hasRole" -ne 1 ]]; then fail "User '$email' does not have the role: $role" else - sed -i "/^$role:$identityId\$/d" "$filename" + sed "/^$role:$identityId\$/d" "$filename" > "$filename.tmp" + cat "$filename".tmp > "$filename" + rm -f "$filename".tmp fi else fail "Unsupported role adjustment operation: $op" fi + return 0 } function createUser() { @@ -370,10 +374,9 @@ EOF reason=$(echo "${response}" | jq ".error.message") [[ $? == 0 ]] && fail "Unable to add user: ${reason}" else + updatePassword "$identityId" addUserRole "$email" "$role" fi - - updatePassword "$identityId" } function updateStatus() { @@ -457,9 +460,10 @@ case "${operation}" in lock validateEmail "$email" - addUserRole "$email" "$role" - syncElastic - echo "Successfully added role to user" + if addUserRole "$email" "$role"; then + syncElastic + echo "Successfully added role to user" + fi ;; "delrole") diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 17b1ad9e0..1d6577e5f 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -124,6 +124,7 @@ syncesusers: - creates: - /opt/so/saltstack/local/salt/elasticsearch/files/users - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles + - /opt/so/conf/soc/soc_users_roles - show_changes: False {% else %} From aed73511e49fc185a0c012f6db24e50e550180d5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 20 Sep 2021 09:24:03 -0400 Subject: [PATCH 114/136] file cleanup, comment cleanup --- salt/elasticsearch/config.map.jinja | 5 -- salt/elasticsearch/files/elasticsearch.yml | 72 ---------------------- 2 files changed, 77 deletions(-) delete mode 100644 salt/elasticsearch/files/elasticsearch.yml diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index d0944842c..1ca729143 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,8 +1,3 @@ -# soup needs -# elasticsearch:esclustername pillar move to elasticsearch:config:cluster:name -# move elasticsearch:true_cluster_name to elasticsearch:config:cluster:name if true_cluster enabled -# elasticsearch:node_route_type moved - {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} {% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml deleted file mode 100644 index 518cd74e9..000000000 --- a/salt/elasticsearch/files/elasticsearch.yml +++ /dev/null @@ -1,72 +0,0 @@ -{%- set NODE_ROUTE_TYPE = salt['pillar.get']('elasticsearch:node_route_type', 'hot') %} -{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip') %} -{%- set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} -{%- if TRUECLUSTER is sameas true %} - {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:true_cluster_name') %} -{%- else %} - {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:esclustername') %} -{%- endif %} -{%- set NODE_ROLES = salt['pillar.get']('elasticsearch:node_roles', ['data', 'ingest']) %} -cluster.name: "{{ ESCLUSTERNAME }}" -network.host: 0.0.0.0 -path.logs: /var/log/elasticsearch -action.destructive_requires_name: true -transport.bind_host: 0.0.0.0 -transport.publish_host: {{ grains.host }} -transport.publish_port: 9300 -cluster.routing.allocation.disk.threshold_enabled: true -cluster.routing.allocation.disk.watermark.low: 95% -cluster.routing.allocation.disk.watermark.high: 98% -cluster.routing.allocation.disk.watermark.flood_stage: 98% -xpack.ml.enabled: false -xpack.security.enabled: true -xpack.security.transport.ssl.enabled: true -xpack.security.transport.ssl.verification_mode: none -xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key -xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt -xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ] -xpack.security.http.ssl.enabled: true -xpack.security.http.ssl.client_authentication: none -xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key -xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt -xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt -{% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} -xpack.security.authc: - anonymous: - username: anonymous_user - roles: superuser - authz_exception: true -{% endif %} -node.name: {{ grains.host }} -script.max_compilations_rate: 20000/1m -{%- if TRUECLUSTER is sameas true %} - {%- if grains.role == 'so-manager' %} - {%- if salt['pillar.get']('nodestab', {}) %} -node.roles: [ master, data, remote_cluster_client ] -discovery.seed_hosts: - - {{ grains.master }} - {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - - {{ SN.split('_')|first }} - {%- endfor %} - {%- endif %} - {%- elif grains.role == 'so-managersearch' %} - {%- if salt['pillar.get']('nodestab', {}) %} -node.roles: [ master, data, remote_cluster_client ] -discovery.seed_hosts: - - {{ grains.master }} - {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - - {{ SN.split('_')|first }} - {%- endfor %} - {%- endif %} -node.attr.box_type: {{ NODE_ROUTE_TYPE }} - {%- else %} -node.roles: {{ NODE_ROLES }} -node.attr.box_type: {{ NODE_ROUTE_TYPE }} -discovery.seed_hosts: - - {{ grains.master }} - {%- endif %} -{%- endif %} -{%- if TRUECLUSTER is sameas false %} -node.attr.box_type: {{ NODE_ROUTE_TYPE }} -{%- endif %} -indices.query.bool.max_clause_count: 1500 From 1e5e1c9ef009dacd0ce9aa58f2fb53fed1d4eaf4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 20 Sep 2021 10:42:55 -0400 Subject: [PATCH 115/136] Update soup --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index c8935969d..ee744b7f2 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -385,7 +385,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20 [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50 - [[ "$INSTALLEDVERSION" == 2.3.50 || "$INSTALLEDVERSION" == 2.3.51 || "$INSTALLEDVERSION" == 2.3.52 || "$INSTALLEDVERSION" == 2.3.60 || "$INSTALLEDVERSION" == 2.3.61 || "$INSTALLEDVERSION" == 2.3.70]] && up_2.3.5X_to_2.3.80 + [[ "$INSTALLEDVERSION" == 2.3.50 || "$INSTALLEDVERSION" == 2.3.51 || "$INSTALLEDVERSION" == 2.3.52 || "$INSTALLEDVERSION" == 2.3.60 || "$INSTALLEDVERSION" == 2.3.61 || "$INSTALLEDVERSION" == 2.3.70 ]] && up_2.3.5X_to_2.3.80 true } From a9049eccd4956e017368c4eb41d15564a25cf745 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 20 Sep 2021 13:30:05 -0400 Subject: [PATCH 116/136] Ensure identity ID parm is quoted now that it doesn't have embedded quotes in the value --- salt/common/tools/sbin/so-user | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index df77ed1e6..ac42a73b4 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -367,7 +367,7 @@ EOF [[ $? != 0 ]] && fail "Unable to communicate with Kratos" identityId=$(echo "${response}" | jq -r ".id") - if [[ ${identityId} == "null" ]]; then + if [[ "${identityId}" == "null" ]]; then code=$(echo "${response}" | jq ".error.code") [[ "${code}" == "409" ]] && fail "User already exists" @@ -389,17 +389,17 @@ function updateStatus() { response=$(curl -Ss -L "${kratosUrl}/identities/$identityId") [[ $? != 0 ]] && fail "Unable to communicate with Kratos" - oldConfig=$(echo "select config from identity_credentials where identity_id=${identityId};" | sqlite3 "$databasePath") + oldConfig=$(echo "select config from identity_credentials where identity_id='${identityId}';" | sqlite3 "$databasePath") if [[ "$status" == "locked" ]]; then config=$(echo $oldConfig | sed -e 's/hashed/locked/') - echo "update identity_credentials set config=CAST('${config}' as BLOB) where identity_id=${identityId};" | sqlite3 "$databasePath" + echo "update identity_credentials set config=CAST('${config}' as BLOB) where identity_id='${identityId}';" | sqlite3 "$databasePath" [[ $? != 0 ]] && fail "Unable to lock credential record" - echo "delete from sessions where identity_id=${identityId};" | sqlite3 "$databasePath" + echo "delete from sessions where identity_id='${identityId}';" | sqlite3 "$databasePath" [[ $? != 0 ]] && fail "Unable to invalidate sessions" else config=$(echo $oldConfig | sed -e 's/locked/hashed/') - echo "update identity_credentials set config=CAST('${config}' as BLOB) where identity_id=${identityId};" | sqlite3 "$databasePath" + echo "update identity_credentials set config=CAST('${config}' as BLOB) where identity_id='${identityId}';" | sqlite3 "$databasePath" [[ $? != 0 ]] && fail "Unable to unlock credential record" fi @@ -415,7 +415,7 @@ function updateUser() { identityId=$(findIdByEmail "$email") [[ ${identityId} == "" ]] && fail "User not found" - updatePassword $identityId + updatePassword "$identityId" } function deleteUser() { From 2c8cbf0db1352675e07d60d4a3deda8abbda786e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 21 Sep 2021 09:53:09 -0400 Subject: [PATCH 117/136] Soup Changes for True Clusters --- salt/common/tools/sbin/soup | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index ee744b7f2..817b0a4c1 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -634,6 +634,15 @@ up_2.3.5X_to_2.3.80() { sed -i '/ index_settings:/a \\ so-kibana: \n shards: 1 \n warm: 7 \n close: 30 \n delete: 365' /opt/so/saltstack/local/pillar/global.sls sed -i '/ index_settings:/a \\ so-redis: \n shards: 1 \n warm: 7 \n close: 30 \n delete: 365' /opt/so/saltstack/local/pillar/global.sls + # Do some pillar formatting + tc=$(grep -w true_cluster /opt/so/saltstack/local/pillar/global.sls | awk -F: {'print tolower($2)'}| xargs) + + if [[ "$tc" == "true" ]]; then + tcname=$(grep -w true_cluster_name /opt/so/saltstack/local/pillar/global.sls | awk -F: {'print $2'}) + sed -i '/elasticsearch:/a \\ config: \n cluster: \n name: $tcname' /opt/so/saltstack/local/pillar/global.sls + sed -i ' true_cluster_name/d /opt/so/saltstack/local/pillar/global.sls' + else + INSTALLEDVERSION=2.3.80 } From ed28e4d0000bc0593686fad1eee51f0da932e2d4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 21 Sep 2021 09:55:49 -0400 Subject: [PATCH 118/136] Soup Changes for True Clusters --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 817b0a4c1..6460e04fa 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -640,8 +640,8 @@ up_2.3.5X_to_2.3.80() { if [[ "$tc" == "true" ]]; then tcname=$(grep -w true_cluster_name /opt/so/saltstack/local/pillar/global.sls | awk -F: {'print $2'}) sed -i '/elasticsearch:/a \\ config: \n cluster: \n name: $tcname' /opt/so/saltstack/local/pillar/global.sls - sed -i ' true_cluster_name/d /opt/so/saltstack/local/pillar/global.sls' - else + sed -i '/ true_cluster_name/d' /opt/so/saltstack/local/pillar/global.sls + fi INSTALLEDVERSION=2.3.80 } From 19efa493ad15c8eb768d093cebf1adfb3407eba5 Mon Sep 17 00:00:00 2001 From: Pete Date: Tue, 21 Sep 2021 11:21:07 -0400 Subject: [PATCH 119/136] missing dollarsign --- salt/common/tools/sbin/so-raid-status | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-raid-status b/salt/common/tools/sbin/so-raid-status index e817554d3..dbc8ba783 100755 --- a/salt/common/tools/sbin/so-raid-status +++ b/salt/common/tools/sbin/so-raid-status @@ -90,7 +90,7 @@ check_software_raid {%- endif %} if [[ -n $SWRAID ]]; then - if [[ $SWRAID == '0' && BOSSRAID == '0' ]]; then + if [[ $SWRAID == '0' && $BOSSRAID == '0' ]]; then RAIDSTATUS=0 else RAIDSTATUS=1 From 5c2be487f58529ae96a80fcb22294dc0d496113b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 21 Sep 2021 13:01:40 -0400 Subject: [PATCH 120/136] fi xquotes --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6460e04fa..2c21e134e 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -639,7 +639,7 @@ up_2.3.5X_to_2.3.80() { if [[ "$tc" == "true" ]]; then tcname=$(grep -w true_cluster_name /opt/so/saltstack/local/pillar/global.sls | awk -F: {'print $2'}) - sed -i '/elasticsearch:/a \\ config: \n cluster: \n name: $tcname' /opt/so/saltstack/local/pillar/global.sls + sed -i "/elasticsearch:/a \\ config: \n cluster: \n name: $tcname" /opt/so/saltstack/local/pillar/global.sls sed -i '/ true_cluster_name/d' /opt/so/saltstack/local/pillar/global.sls fi From d82e91f69ed02e56a3354c24d7b9ab5b9fec70ff Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 21 Sep 2021 13:54:45 -0400 Subject: [PATCH 121/136] match elasticsearch at beginning of line --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 2c21e134e..26076769b 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -639,7 +639,7 @@ up_2.3.5X_to_2.3.80() { if [[ "$tc" == "true" ]]; then tcname=$(grep -w true_cluster_name /opt/so/saltstack/local/pillar/global.sls | awk -F: {'print $2'}) - sed -i "/elasticsearch:/a \\ config: \n cluster: \n name: $tcname" /opt/so/saltstack/local/pillar/global.sls + sed -i "/^elasticsearch:/a \\ config: \n cluster: \n name: $tcname" /opt/so/saltstack/local/pillar/global.sls sed -i '/ true_cluster_name/d' /opt/so/saltstack/local/pillar/global.sls fi From f4186feffaebebf7be244131ad44bd352f16c601 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 21 Sep 2021 15:40:49 -0400 Subject: [PATCH 122/136] move node_route_type --- salt/common/tools/sbin/soup | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 26076769b..7fb345aa7 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -641,6 +641,13 @@ up_2.3.5X_to_2.3.80() { tcname=$(grep -w true_cluster_name /opt/so/saltstack/local/pillar/global.sls | awk -F: {'print $2'}) sed -i "/^elasticsearch:/a \\ config: \n cluster: \n name: $tcname" /opt/so/saltstack/local/pillar/global.sls sed -i '/ true_cluster_name/d' /opt/so/saltstack/local/pillar/global.sls + sed -i '/ esclustername/d' /opt/so/saltstack/local/pillar/global.sls + + for file in /opt/so/saltstack/local/pillar/minions/*.sls; do + noderoutetype=$(grep -w node_route_type $file | awk -F: {'print $2'}) + sed -i "/^elasticsearch:/a \\ config: \n node: \n attr: \n box_type: $noderoutetype" $file + sed -i '/ node_route_type/d' $file + done fi INSTALLEDVERSION=2.3.80 From dae41d279a43437aa7357182cc5095b59d1ea5e7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 22 Sep 2021 08:25:55 -0400 Subject: [PATCH 123/136] Prevent emails addresses from having uppercase characters --- salt/common/tools/sbin/so-user | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index ac42a73b4..f7604d298 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -99,8 +99,7 @@ function validatePassword() { len=$(expr length "$password") if [[ $len -lt 6 ]]; then - echo "Password does not meet the minimum requirements" - exit 2 + fail "Password does not meet the minimum requirements" fi check_password_and_exit "$password" } @@ -109,8 +108,11 @@ function validateEmail() { email=$1 # (?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\]) if [[ ! "$email" =~ ^[[:alnum:]._%+-]+@[[:alnum:].-]+\.[[:alpha:]]{2,}$ ]]; then - echo "Email address is invalid" - exit 3 + fail "Email address is invalid" + fi + + if [[ "$email" =~ [A-Z] ]]; then + fail "Email addresses cannot contain uppercase letters" fi } From edf75255cf7a4d375ec110dd97e8c7e6909fe040 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 22 Sep 2021 12:01:32 -0400 Subject: [PATCH 124/136] ignore manager pillar file for noderoutetype --- salt/common/tools/sbin/soup | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 7fb345aa7..49ccc2538 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -644,9 +644,14 @@ up_2.3.5X_to_2.3.80() { sed -i '/ esclustername/d' /opt/so/saltstack/local/pillar/global.sls for file in /opt/so/saltstack/local/pillar/minions/*.sls; do - noderoutetype=$(grep -w node_route_type $file | awk -F: {'print $2'}) - sed -i "/^elasticsearch:/a \\ config: \n node: \n attr: \n box_type: $noderoutetype" $file - sed -i '/ node_route_type/d' $file + if [[ ${file} != *"manager.sls"* ]]; then + noderoutetype=$(grep -w node_route_type $file | awk -F: {'print $2'}) + if [ -n "$noderoutetype" ]; then + sed -i "/^elasticsearch:/a \\ config: \n node: \n attr: \n box_type: $noderoutetype" $file + sed -i '/ node_route_type/d' $file + noderoutetype='' + fi + fi done fi From cdb9dcbaece138d73bbd78c6e5c02f69e110cee8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 22 Sep 2021 15:07:36 -0400 Subject: [PATCH 125/136] notify of custom es config --- salt/common/tools/sbin/soup | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 49ccc2538..cd38a046f 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -27,6 +27,7 @@ SOUP_LOG=/root/soup.log INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log WHATWOULDYOUSAYYAHDOHERE=soup whiptail_title='Security Onion UPdater' +NOTIFYCUSTOMELASTICCONFIG=false check_err() { local exit_code=$1 @@ -655,6 +656,11 @@ up_2.3.5X_to_2.3.80() { done fi + # check for local es config to inform user that the config in local is now ignored and those options need to be placed in the pillar + if [ -f "/opt/so/saltstack/local/salt/elasticsearch/files/elasticsearch.yml" ]; then + NOTIFYCUSTOMELASTICCONFIG=true + fi + INSTALLEDVERSION=2.3.80 } @@ -1059,6 +1065,17 @@ EOF fi fi + if [ "$NOTIFYCUSTOMELASTICCONFIG" = true ] ; then + cat < EOF + + +A custom Elasticsearch configuration has been found at /opt/so/saltstack/local/elasticsearch/files/elasticsearch.yml. This file is no longer referenced in Security Onion verions >= 2.3.80. + +In order to customize Elasticsearch, please see https://docs.securityonion.net/en/2.3/elasticsearch.html. + + + fi + echo "### soup has been served at $(date) ###" } From 52c1298b9b290ed1dc18e79ec59d945121ec70c6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 22 Sep 2021 15:16:07 -0400 Subject: [PATCH 126/136] notify of custom es config --- salt/common/tools/sbin/soup | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index cd38a046f..d53dcd73c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -1066,13 +1066,15 @@ EOF fi if [ "$NOTIFYCUSTOMELASTICCONFIG" = true ] ; then - cat < EOF + + cat << EOF -A custom Elasticsearch configuration has been found at /opt/so/saltstack/local/elasticsearch/files/elasticsearch.yml. This file is no longer referenced in Security Onion verions >= 2.3.80. +A custom Elasticsearch configuration has been found at /opt/so/saltstack/local/elasticsearch/files/elasticsearch.yml. This file is no longer referenced in Security Onion versions >= 2.3.80. -In order to customize Elasticsearch, please see https://docs.securityonion.net/en/2.3/elasticsearch.html. +If you still need those customizations, you'll need to manually migrate them to the new Elasticsearch config as shown at https://docs.securityonion.net/en/2.3/elasticsearch.html. +EOF fi From 73dacdcbff8ef8594168ddd31499d3aca9cbd8c2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 23 Sep 2021 09:52:23 -0400 Subject: [PATCH 127/136] Add logscan to images for pull during soup if it's enabled --- salt/common/tools/sbin/so-image-common | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 9b6e2174a..0b0f89698 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -17,6 +17,7 @@ # NOTE: This script depends on so-common IMAGEREPO=security-onion-solutions +STATUS_CONF='/opt/so/conf/so-status/so-status.conf' # shellcheck disable=SC2120 container_list() { @@ -137,6 +138,11 @@ update_docker_containers() { if [[ $result -eq 0 ]]; then cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1 fi + + # If downloading for soup, check if any optional images need to be pulled + if [[ $CURLTYPE == 'soup' ]]; then + grep -q "so-logscan" "$STATUS_CONF" && TRUSTED_CONTAINERS+=("so-logscan") + fi # Download the containers from the interwebs for i in "${TRUSTED_CONTAINERS[@]}" From b8b1867e525a6974b0e95644d822e5401e50580f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 23 Sep 2021 12:43:23 -0400 Subject: [PATCH 128/136] Tell user what soup is doing at end of upgrade --- salt/common/tools/sbin/soup | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d53dcd73c..828f5c741 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -1016,6 +1016,7 @@ main() { [[ $is_airgap -eq 0 ]] && unmount_update thehive_maint + echo "Checking the number of minions." NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l) if [[ $UPGRADESALT -eq 1 ]] && [[ $NUM_MINIONS -gt 1 ]]; then if [[ $is_airgap -eq 0 ]]; then @@ -1026,8 +1027,10 @@ main() { fi fi + echo "Checking for local modifications." check_local_mods + echo "Checking sudoers file." check_sudoers if [[ -n $lsl_msg ]]; then From a01d49981c5d39fa9bb7334767000f2d889ecb48 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 23 Sep 2021 12:45:44 -0400 Subject: [PATCH 129/136] Redirect thehive/cortex migrate curl output to soup log --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 828f5c741..6cdeda97f 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -704,8 +704,8 @@ thehive_maint() { done if [ "$THEHIVE_CONNECTED" == "yes" ]; then echo "Migrating thehive databases if needed." - curl -v -k -XPOST -L "https://localhost/thehive/api/maintenance/migrate" - curl -v -k -XPOST -L "https://localhost/cortex/api/maintenance/migrate" + curl -v -k -XPOST -L "https://localhost/thehive/api/maintenance/migrate" &> "$SOUP_LOG" + curl -v -k -XPOST -L "https://localhost/cortex/api/maintenance/migrate" &> "$SOUP_LOG" fi } From 8a9dcb7fdbe4191b53b0c13870994fb46e715ac2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 23 Sep 2021 12:47:22 -0400 Subject: [PATCH 130/136] Fix "upgrade to" message Only specify "to" version and change when the upgrade message occurs --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6cdeda97f..180fa7c38 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -992,8 +992,6 @@ main() { set +e salt-call state.highstate -l info queue=True set -e - echo "" - echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete." echo "" echo "Stopping Salt Master to remove ACL" @@ -1016,6 +1014,8 @@ main() { [[ $is_airgap -eq 0 ]] && unmount_update thehive_maint + echo "" + echo "Upgrade to $NEWVERSION complete." echo "Checking the number of minions." NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l) if [[ $UPGRADESALT -eq 1 ]] && [[ $NUM_MINIONS -gt 1 ]]; then From b44358fc26a58099c2d449cd998b09f7b3037309 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 23 Sep 2021 12:49:42 -0400 Subject: [PATCH 131/136] Add `set +e` after final upgrade steps and before post-upgrade checks --- salt/common/tools/sbin/soup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 180fa7c38..fd325b8a4 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -1016,6 +1016,10 @@ main() { echo "" echo "Upgrade to $NEWVERSION complete." + + # Everything beyond this is post-upgrade checking, don't fail past this point if something here causes an error + set +e + echo "Checking the number of minions." NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l) if [[ $UPGRADESALT -eq 1 ]] && [[ $NUM_MINIONS -gt 1 ]]; then From 8b905b585d654795cc65b12cdda555d605eeadd3 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 23 Sep 2021 12:55:06 -0400 Subject: [PATCH 132/136] Fix redirect to append --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index fd325b8a4..4ef0d9185 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -704,8 +704,8 @@ thehive_maint() { done if [ "$THEHIVE_CONNECTED" == "yes" ]; then echo "Migrating thehive databases if needed." - curl -v -k -XPOST -L "https://localhost/thehive/api/maintenance/migrate" &> "$SOUP_LOG" - curl -v -k -XPOST -L "https://localhost/cortex/api/maintenance/migrate" &> "$SOUP_LOG" + curl -v -k -XPOST -L "https://localhost/thehive/api/maintenance/migrate" &>> "$SOUP_LOG" + curl -v -k -XPOST -L "https://localhost/cortex/api/maintenance/migrate" &>> "$SOUP_LOG" fi } From f8bea824301b0e75c1f64ec58ce05271b85ef27d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 23 Sep 2021 12:57:08 -0400 Subject: [PATCH 133/136] Make redirect consistent with setup --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 4ef0d9185..caea21866 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -704,8 +704,8 @@ thehive_maint() { done if [ "$THEHIVE_CONNECTED" == "yes" ]; then echo "Migrating thehive databases if needed." - curl -v -k -XPOST -L "https://localhost/thehive/api/maintenance/migrate" &>> "$SOUP_LOG" - curl -v -k -XPOST -L "https://localhost/cortex/api/maintenance/migrate" &>> "$SOUP_LOG" + curl -v -k -XPOST -L "https://localhost/thehive/api/maintenance/migrate" >> "$SOUP_LOG" 2>&1 + curl -v -k -XPOST -L "https://localhost/cortex/api/maintenance/migrate" >> "$SOUP_LOG" 2>&1 fi } From 275163f85d74a54ff0ab126e6a58baf93e34f88d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 27 Sep 2021 07:36:54 -0400 Subject: [PATCH 134/136] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b64c72a34..580590829 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -## Security Onion 2.3.70 +## Security Onion 2.3.80 -Security Onion 2.3.70 is here! +Security Onion 2.3.80 is here! ## Screenshots From b069377c8a4ab1030292a5016d78d2fd0e848f66 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 27 Sep 2021 10:13:42 -0400 Subject: [PATCH 135/136] 2.3.80 --- VERIFY_ISO.md | 22 +++++++++++----------- sigs/securityonion-2.3.80.iso.sig | Bin 0 -> 543 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-2.3.80.iso.sig diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index f6cc31508..dae008a8f 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -1,18 +1,18 @@ -### 2.3.70-WAZUH ISO image built on 2021/08/30 +### 2.3.80 ISO image built on 2021/08/30 ### Download and Verify -2.3.70-WAZUH ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.3.70-WAZUH.iso +2.3.80 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.3.80.iso -MD5: CEDEF3C38089896C252F9E3C75F7CB15 -SHA1: FB420115C72DABDEB87C8B27F26E862C94628057 -SHA256: CC3E75A97163E9CD255DA0D9C3EB11922FA045651827F291025398943C1BC230 +MD5: 24F38563860416F4A8ABE18746913E14 +SHA1: F923C005F54EA2A17AB225ADA0DA46042707AAD9 +SHA256: 8E95D10AF664D9A406C168EC421D943CB23F0D0C1813C6C2DBA9B4E131984018 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-WAZUH.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.80.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS @@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-WAZUH.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.80.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.3.70-WAZUH.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.3.80.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.3.70-WAZUH.iso.sig securityonion-2.3.70-WAZUH.iso +gpg --verify securityonion-2.3.80.iso.sig securityonion-2.3.80.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Mon 30 Aug 2021 06:13:14 PM EDT using RSA key ID FE507013 +gpg: Signature made Mon 27 Sep 2021 08:55:01 AM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.3.80.iso.sig b/sigs/securityonion-2.3.80.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..4fa76de2e248663018bbb18c9494692be0d262d7 GIT binary patch literal 543 zcmV+)0^t3L0vrSY0RjL91p;AFza;<)2@re`V7LBIa1%185C42D7oKYXdOn02`!}QM z{*}VbE0))c!-F>4rLTz?`f}3am@^@Ms&As|xYR}E2Lqirc*45Rw5I~$UHyk4Ug9U3 z>BzgrCSICSp<|3-b%ftu}USBUI}2L4N&rBy1i6xFY4oY`72)xOu@ z){UdfrS~5@m-I2niyOU45*xGg&r_nyVWFego;#ROH?v9Ka#!9a2NsyEOuj; z_11h7PGq;5yj4*M75%|q&tj21D!pvGH-TjwGzNZ(#8Hi$<}mW0j8~?%;w#@2KT16v z^V<1Io)(!~R*kZpYJRhi?~`?Jg2?^N-49knv04lH%x$cyb}bBn!zx2I;3aDeDq?2p zw5Whgqc~6O7B6I&!4CV(;P(krWxHR*4BR&%Ap+Yn7;LlLvRN?;*jp`9W}AxS53L^Y z30$k>yLfzs3qeyd7I1}rXCkUIaa4jNVNJakIc=XT@MOA?Swu!Dp?LU;)~A*&`x7+n zvEo73A7ZP%F0-JMvJjkHK7x5o=))D`1KLEO>!d9vR7{gRwPhyS9ehYDwjA!*c2{`k h@5q%SJRbyLK4c~-4J8SqZ@nBfxC+)`(s7xyhC9cm1#18R literal 0 HcmV?d00001 From b1d0e3e93fe32613ef4dee5e81bf80d26be20dd5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 27 Sep 2021 12:32:45 -0400 Subject: [PATCH 136/136] 2.3.80 --- VERIFY_ISO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index dae008a8f..1a8c028f3 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -1,4 +1,4 @@ -### 2.3.80 ISO image built on 2021/08/30 +### 2.3.80 ISO image built on 2021/09/27