From 0681d29bb085c60604ae023d3e0f71c6a165bd8c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 25 Aug 2021 10:23:06 -0400 Subject: [PATCH 01/31] starting es pillarization --- salt/elasticsearch/config.map.jinja | 5 + salt/elasticsearch/defaults.yaml | 1112 +++++++++++++++++++++++++++ salt/elasticsearch/init.sls | 11 + 3 files changed, 1128 insertions(+) create mode 100644 salt/elasticsearch/config.map.jinja create mode 100644 salt/elasticsearch/defaults.yaml diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja new file mode 100644 index 000000000..e6860e699 --- /dev/null +++ b/salt/elasticsearch/config.map.jinja @@ -0,0 +1,5 @@ +{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG %} + +{% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} + {% do ESCONFIG.elasticsearch.defaults.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} +{% endif %} diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml new file mode 100644 index 000000000..8fc244d6c --- /dev/null +++ b/salt/elasticsearch/defaults.yaml @@ -0,0 +1,1112 @@ +elasticsearch: + es_port: 9200 + esheap: 4049m + esclustername: default-cluster-name + log_size_limit: 95 #used for curator + + index_settings: + so-beats: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-firewall: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-flow: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-ids: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-import: + shards: 1 + warm: 7 + close: 73000 + delete: 73001 + so-osquery: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-ossec: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-strelka: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-syslog: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-zeek: + shards: 5 + warm: 7 + close: 45 + delete: 365 + + + + persistent: + cluster: + remote: + default-cluster-name: + seeds: + - 127.0.0.1:9300 + transient: + {} + defaults: + cluster: + max_voting_config_exclusions: 10 + auto_shrink_voting_configuration: true + election: + duration: 500ms + initial_timeout: 100ms + max_timeout: 10s + back_off_time: 100ms + strategy: supports_voting_only + no_master_block: write + persistent_tasks: + allocation: + enable: all + recheck_interval: 30s + blocks: + read_only_allow_delete: false + read_only: false + remote: + node: + attr: + initial_connect_timeout: 30s + connect: true + connections_per_cluster: 3 + follower_lag: + timeout: 90000ms + routing: + use_adaptive_replica_selection: true + rebalance: + enable: all + allocation: + node_concurrent_incoming_recoveries: 2 + include: + _tier: + node_initial_primaries_recoveries: 4 + same_shard: + host: false + total_shards_per_node: -1 + require: + _tier: + shard_state: + reroute: + priority: NORMAL + type: balanced + disk: + threshold_enabled: true + watermark: + flood_stage.frozen.max_headroom: 20GB + flood_stage: 98% + high: 98% + low: 95% + enable_for_single_data_node: false + flood_stage.frozen: 95% + include_relocations: true + reroute_interval: 60s + awareness: + attributes: [] + balance: + index: 0.55 + threshold: 1.0 + shard: 0.45 + enable: all + node_concurrent_outgoing_recoveries: 2 + allow_rebalance: indices_all_active + cluster_concurrent_rebalance: 2 + node_concurrent_recoveries: 2 + exclude: + _tier: + indices: + tombstones: + size: 500 + close: + enable: true + max_shards_per_node.frozen: 3000 + nodes: + reconnect_interval: 10s + service: + slow_master_task_logging_threshold: 10s + slow_task_logging_threshold: 30s + publish: + timeout: 30000ms + info_timeout: 10000ms + name: default-cluster-name + fault_detection: + leader_check: + interval: 1000ms + timeout: 10000ms + retry_count: 3 + follower_check: + interval: 1000ms + timeout: 10000ms + retry_count: 3 + join: + timeout: 60000ms + max_shards_per_node: 1000 + initial_master_nodes: [] + snapshot: + info: + max_concurrent_fetches: 5 + info: + update: + interval: 30s + timeout: 15s + stack: + templates: + enabled: true + logger: + level: INFO + bootstrap: + memory_lock: false + system_call_filter: true + ctrlhandler: true + processors: 8 + ingest: + user_agent: + cache_size: 1000 + geoip: + cache_size: 1000 + downloader: + enabled: false + endpoint: https://geoip.elastic.co/v1/database + poll: + interval: 3d + grok: + watchdog: + max_execution_time: 1s + interval: 1s + network: + host: + - 0.0.0.0 + tcp: + reuse_address: true + keep_count: -1 + connect_timeout: 30s + keep_interval: -1 + no_delay: true + keep_alive: true + receive_buffer_size: -1b + keep_idle: -1 + send_buffer_size: -1b + bind_host: + - 0.0.0.0 + server: true + breaker: + inflight_requests: + limit: 100% + overhead: 2.0 + publish_host: + - 0.0.0.0 + pidfile: + path: + data: [] + logs: /var/log/elasticsearch + shared_data: + home: /usr/share/elasticsearch + repo: [] + search: + default_search_timeout: -1 + highlight: + term_vector_multi_value: true + default_allow_partial_results: true + max_open_scroll_context: 500 + max_buckets: 65536 + low_level_cancellation: true + allow_expensive_queries: true + keep_alive_interval: 1m + remote: + node: + attr: + initial_connect_timeout: 30s + connect: true + connections_per_cluster: 3 + default_keep_alive: 5m + max_keep_alive: 24h + aggs: + rewrite_to_filter_by_filter: true + security: + manager: + filter_bad_defaults: true + transform: + task_thread_pool: + queue_size: 4 + size: 4 + ccr: + wait_for_metadata_timeout: 60s + indices: + recovery: + recovery_activity_timeout: 60s + chunk_size: 1mb + internal_action_timeout: 60s + max_bytes_per_sec: 40mb + max_concurrent_file_chunks: 5 + auto_follow: + wait_for_metadata_timeout: 60s + repositories: + fs: + compress: false + chunk_size: 9223372036854775807b + location: + url: + supported_protocols: + - http + - https + - ftp + - file + - jar + allowed_urls: [] + url: http: + action: + auto_create_index: true + search: + shard_count: + limit: 9223372036854775807 + destructive_requires_name: true + client: + type: node + transport: + ignore_cluster_name: false + nodes_sampler_interval: 5s + sniff: false + ping_timeout: 5s + enrich: + max_force_merge_attempts: 3 + cleanup_period: 15m + fetch_size: 10000 + coordinator_proxy: + max_concurrent_requests: 8 + max_lookups_per_request: 128 + queue_capacity: 1024 + max_concurrent_policy_executions: 50 + xpack: + flattened: + enabled: true + watcher: + execution: + scroll: + size: 0 + timeout: + default_throttle_period: 5s + internal: + ops: + bulk: + default_timeout: + index: + default_timeout: + search: + default_timeout: + thread_pool: + queue_size: 1000 + size: 40 + index: + rest: + direct_access: + use_ilm_index_management: true + history: + cleaner_service: + enabled: true + trigger: + schedule: + ticker: + tick_interval: 500ms + enabled: true + input: + search: + default_timeout: + encrypt_sensitive_data: false + transform: + search: + default_timeout: + stop: + timeout: 30s + watch: + scroll: + size: 0 + bulk: + concurrent_requests: 0 + flush_interval: 1s + size: 1mb + actions: 1 + actions: + bulk: + default_timeout: + index: + default_timeout: + eql: + enabled: true + data_frame: + enabled: true + ilm: + enabled: true + monitoring: + migration: + decommission_alerts: false + collection: + cluster: + stats: + timeout: 10s + node: + stats: + timeout: 10s + indices: [] + ccr: + stats: + timeout: 10s + enrich: + stats: + timeout: 10s + index: + stats: + timeout: 10s + recovery: + active_only: false + timeout: 10s + interval: 10s + enabled: false + ml: + job: + stats: + timeout: 10s + history: + duration: 168h + elasticsearch: + collection: + enabled: true + enabled: true + graph: + enabled: true + searchable: + snapshot: + allocate_on_rolling_restart: false + cache: + range_size: 32mb + sync: + max_files: 10000 + interval: 60s + shutdown_timeout: 10s + recovery_range_size: 128kb + shared_cache: + recovery_range_size: 128kb + region_size: 16mb + size: 0 + min_time_delta: 60s + decay: + interval: 60s + size.max_headroom: -1 + range_size: 16mb + max_freq: 100 + rollup: + enabled: true + task_thread_pool: + queue_size: -1 + size: 1 + sql: + enabled: true + searchable_snapshots: + cache_fetch_async_thread_pool: + core: 0 + max: 24 + keep_alive: 30s + cache_prewarming_thread_pool: + core: 0 + max: 16 + keep_alive: 30s + license: + upload: + types: + - standard + - gold + - platinum + - enterprise + - trial + self_generated: + type: basic + logstash: + enabled: true + notification: + pagerduty: + default_account: + email: + default_account: + html: + sanitization: + allow: + - body + - head + - _tables + - _links + - _blocks + - _formatting + - img:embedded + disallow: [] + enabled: true + reporting: + retries: 40 + warning: + enabled: true + interval: 15s + jira: + default_account: + slack: + default_account: + security: + operator_privileges: + enabled: false + dls_fls: + enabled: true + dls: + bitset: + cache: + size: 10% + ttl: 2h + transport: + filter: + allow: [] + deny: [] + enabled: true + ssl: + enabled: true + ssl: + diagnose: + trust: true + enabled: true + crypto: + thread_pool: + queue_size: 1000 + size: 4 + filter: + always_allow_bound_address: true + encryption: + algorithm: AES/CTR/NoPadding + audit: + enabled: false + logfile: + emit_node_id: true + emit_node_host_name: false + emit_node_name: false + events: + emit_request_body: false + include: + - ACCESS_DENIED + - ACCESS_GRANTED + - ANONYMOUS_ACCESS_DENIED + - AUTHENTICATION_FAILED + - CONNECTION_DENIED + - TAMPERED_REQUEST + - RUN_AS_DENIED + - RUN_AS_GRANTED + - SECURITY_CONFIG_CHANGE + exclude: + [] + emit_node_host_address: false + authc: + password_hashing: + algorithm: bcrypt + success_cache: + size: 10000 + enabled: true + expire_after_access: 1h + api_key: + doc_cache: + ttl: 5m + cache: + hash_algo: ssha256 + max_keys: 10000 + ttl: 24h + delete: + interval: 24h + timeout: -1 + enabled: false + hashing: + algorithm: pbkdf2 + anonymous: + authz_exception: true + roles: + - superuser + username: anonymous_user + run_as: + enabled: true + reserved_realm: + enabled: true + service_token: + cache: + hash_algo: ssha256 + max_tokens: 100000 + ttl: 20m + token: + delete: + interval: 30m + timeout: -1 + enabled: false + thread_pool: + queue_size: 1000 + size: 1 + timeout: 20m + fips_mode: + enabled: false + encryption_key: + length: 128 + algorithm: AES + http: + filter: + allow: [] + deny: [] + enabled: true + ssl: + enabled: true + automata: + max_determinized_states: 100000 + cache: + size: 10000 + ttl: 48h + enabled: true + user: null + authz: + store: + privileges: + cache: + ttl: 24h + max_size: 10000 + roles: + index: + cache: + ttl: 20m + max_size: 10000 + cache: + max_size: 10000 + negative_lookup_cache: + max_size: 10000 + field_permissions: + cache: + max_size_in_bytes: 104857600 + transform: + num_transform_failure_retries: 10 + enabled: true + vectors: + enabled: true + ccr: + enabled: true + ccr_thread_pool: + queue_size: 100 + size: 32 + idp: + privileges: + application: + cache: + size: 100 + ttl: 90m + metadata: + signing: + keystore: + alias: + slo_endpoint: + post: https: + redirect: https: + defaults: + nameid_format: urn:oasis:names:tc:SAML:2.0:nameid-format:transient + authn_expiry: 5m + allowed_nameid_formats: + - urn:oasis:names:tc:SAML:2.0:nameid-format:transient + contact: + given_name: + email: + surname: + organization: + display_name: + name: + url: http: + sso_endpoint: + post: https: + redirect: https: + entity_id: + signing: + keystore: + alias: + sp: + cache: + size: 1000 + ttl: 60m + wildcard: + path: wildcard_services.json + enabled: false + slm: + enabled: true + enrich: + enabled: true + http: + default_connection_timeout: 10s + proxy: + host: + scheme: + port: 0 + whitelist: + - * + default_read_timeout: 10s + max_response_size: 10mb + autoscaling: + memory: + monitor: + timeout: 15s + ml: + max_anomaly_records: 500 + enable_config_migration: true + max_open_jobs: 512 + min_disk_space_off_heap: 5gb + use_auto_machine_memory_percent: false + inference_model: + cache_size: 40% + time_to_live: 5m + nightly_maintenance_requests_per_second: -1.0 + node_concurrent_job_allocations: 2 + max_model_memory_limit: 0b + enabled: false + max_lazy_ml_nodes: 0 + max_ml_node_size: 0b + max_machine_memory_percent: 30 + persist_results_max_retries: 20 + autodetect_process: true + max_inference_processors: 50 + process_connect_timeout: 10s + rest: + action: + multi: + allow_explicit_index: true + cache: + recycler: + page: + limit: + heap: 10% + type: CONCURRENT + weight: + longs: 1.0 + ints: 1.0 + bytes: 1.0 + objects: 0.1 + async_search: + index_cleanup_interval: 1h + reindex: + remote: + whitelist: [] + resource: + reload: + enabled: true + interval: + low: 60s + high: 5s + medium: 30s + thread_pool: + force_merge: + queue_size: -1 + size: 1 + fetch_shard_started: + core: 1 + max: 16 + keep_alive: 5m + listener: + queue_size: -1 + size: 4 + refresh: + core: 1 + max: 4 + keep_alive: 5m + system_write: + queue_size: 1000 + size: 4 + generic: + core: 4 + max: 128 + keep_alive: 30s + warmer: + core: 1 + max: 4 + keep_alive: 5m + search: + max_queue_size: 1000 + queue_size: 1000 + size: 13 + auto_queue_frame_size: 2000 + target_response_time: 1s + min_queue_size: 1000 + fetch_shard_store: + core: 1 + max: 16 + keep_alive: 5m + flush: + core: 1 + max: 4 + keep_alive: 5m + management: + core: 1 + max: 5 + keep_alive: 5m + analyze: + queue_size: 16 + size: 1 + get: + queue_size: 1000 + size: 8 + system_read: + queue_size: 2000 + size: 4 + estimated_time_interval: 200ms + write: + queue_size: 10000 + size: 8 + snapshot: + core: 1 + max: 4 + keep_alive: 5m + search_throttled: + max_queue_size: 100 + queue_size: 100 + size: 1 + auto_queue_frame_size: 200 + target_response_time: 1s + min_queue_size: 100 + index: + codec: default + recovery: + type: + store: + type: + fs: + fs_lock: native + preload: [] + snapshot: + uncached_chunk_size: -1b + cache: + excluded_file_types: [] + monitor: + jvm: + gc: + enabled: true + overhead: + warn: 50 + debug: 10 + info: 25 + refresh_interval: 1s + refresh_interval: 1s + process: + refresh_interval: 1s + os: + refresh_interval: 1s + fs: + health: + enabled: true + refresh_interval: 120s + slow_path_logging_threshold: 5s + refresh_interval: 1s + runtime_fields: + grok: + watchdog: + max_execution_time: 1s + interval: 1s + transport: + tcp: + reuse_address: true + keep_count: -1 + connect_timeout: 30s + keep_interval: -1 + compress: false + port: 9300-9400 + no_delay: true + keep_alive: true + receive_buffer_size: -1b + keep_idle: -1 + send_buffer_size: -1b + bind_host: + - 0.0.0.0 + connect_timeout: 30s + compress: false + ping_schedule: -1 + connections_per_node: + recovery: 2 + state: 1 + bulk: 3 + reg: 6 + ping: 1 + tracer: + include: [] + exclude: + - internal:discovery/zen/fd* + - internal:coordination/fault_detection/* + - cluster:monitor/nodes/liveness + type: security4 + slow_operation_logging_threshold: 5s + type.default: netty4 + features: + x-pack: true + port: 9300-9400 + host: [] + publish_port: 9300 + tcp_no_delay: true + publish_host: {{ grains.host }} + netty: + receive_predictor_size: 64kb + receive_predictor_max: 64kb + worker_count: 8 + receive_predictor_min: 64kb + boss_count: 1 + script: + allowed_contexts: [] + max_compilations_rate: 20000/1m + cache: + max_size: 100 + expire: 0ms + painless: + regex: + enabled: limited + limit-factor: 6 + max_size_in_bytes: 65535 + allowed_types: [] + disable_max_compilations_rate: false + indexing_pressure: + memory: + limit: 10% + node: + data: true + roles: + - data_frozen + - data_warm + - transform + - data + - remote_cluster_client + - data_cold + - data_content + - data_hot + - ingest + - master + max_local_storage_nodes: 1 + processors: 8 + store: + allow_mmap: true + ingest: true + master: true + pidfile: + transform: true + remote_cluster_client: true + enable_lucene_segment_infos_trace: false + local_storage: true + name: {{ grains.host }} + id: + seed: 0 + voting_only: false + attr: + transform: + node: true + xpack: + installed: true + box_type: hot + portsfile: false + ml: true + indices: + replication: + retry_timeout: 60s + initial_retry_backoff_bound: 50ms + cache: + cleanup_interval: 1m + mapping: + dynamic_timeout: 30s + max_in_flight_updates: 10 + memory: + interval: 5s + max_index_buffer_size: -1 + shard_inactive_time: 5m + index_buffer_size: 10% + min_index_buffer_size: 48mb + breaker: + request: + limit: 60% + type: memory + overhead: 1.0 + total: + limit: 95% + use_real_memory: true + accounting: + limit: 100% + overhead: 1.0 + fielddata: + limit: 40% + type: memory + overhead: 1.03 + type: hierarchy + query: + bool: + max_nested_depth: 20 + max_clause_count: 1500 + query_string: + analyze_wildcard: false + allowLeadingWildcard: true + id_field_data: + enabled: true + recovery: + recovery_activity_timeout: 1800000ms + retry_delay_network: 5s + internal_action_timeout: 15m + retry_delay_state_sync: 500ms + internal_action_long_timeout: 1800000ms + max_concurrent_operations: 1 + max_bytes_per_sec: 40mb + max_concurrent_file_chunks: 2 + requests: + cache: + size: 1% + expire: 0ms + store: + delete: + shard: + timeout: 30s + analysis: + hunspell: + dictionary: + ignore_case: false + lazy: false + queries: + cache: + count: 10000 + size: 10% + all_segments: false + lifecycle: + history_index_enabled: true + poll_interval: 10m + step: + master_timeout: 30s + fielddata: + cache: + size: -1b + plugin: + mandatory: [] + slm: + minimum_interval: 15m + retention_schedule: 0 30 1 * * ? + retention_duration: 1h + history_index_enabled: true + discovery: + seed_hosts: [] + unconfigured_bootstrap_timeout: 3s + request_peers_timeout: 3000ms + zen: + commit_timeout: 30s + no_master_block: write + join_retry_delay: 100ms + join_retry_attempts: 3 + ping: + unicast: + concurrent_connects: 10 + hosts: [] + hosts.resolve_timeout: 5s + master_election: + ignore_non_master_pings: false + wait_for_joins_timeout: 30000ms + send_leave_request: true + ping_timeout: 3s + bwc_ping_timeout: 3s + join_timeout: 60000ms + publish_diff: + enable: true + publish: + max_pending_cluster_states: 25 + minimum_master_nodes: -1 + unsafe_rolling_upgrades_enabled: true + hosts_provider: [] + publish_timeout: 30s + fd: + connect_on_network_disconnect: false + ping_interval: 1s + ping_retries: 3 + register_connection_listener: true + ping_timeout: 30s + max_pings_from_another_master: 3 + initial_state_timeout: 30s + cluster_formation_warning_timeout: 10000ms + seed_providers: [] + type: single-node + seed_resolver: + max_concurrent_resolvers: 10 + timeout: 5s + find_peers_interval: 1000ms + probe: + connect_timeout: 30s + handshake_timeout: 30s + http: + cors: + max-age: 1728000 + allow-origin: + allow-headers: X-Requested-With,Content-Type,Content-Length + allow-credentials: false + allow-methods: OPTIONS,HEAD,GET,POST,PUT,DELETE + enabled: false + max_chunk_size: 8kb + compression_level: 3 + max_initial_line_length: 4kb + type: security4 + pipelining: + max_events: 10000 + type.default: netty4 + content_type: + required: true + host: [] + publish_port: -1 + read_timeout: 0ms + max_content_length: 100mb + netty: + receive_predictor_size: 64kb + max_composite_buffer_components: 69905 + worker_count: 0 + tcp: + reuse_address: true + keep_count: -1 + keep_interval: -1 + no_delay: true + keep_alive: true + receive_buffer_size: -1b + keep_idle: -1 + send_buffer_size: -1b + bind_host: [] + client_stats: + enabled: true + reset_cookies: false + max_warning_header_count: -1 + tracer: + include: [] + exclude: [] + max_warning_header_size: -1b + detailed_errors: + enabled: true + port: 9200-9300 + max_header_size: 8kb + tcp_no_delay: true + compression: false + publish_host: [] + gateway: + recover_after_master_nodes: 0 + expected_nodes: -1 + recover_after_data_nodes: -1 + expected_data_nodes: -1 + write_dangling_indices_info: true + slow_write_logging_threshold: 10s + recover_after_time: 0ms + expected_master_nodes: -1 + recover_after_nodes: -1 + auto_import_dangling_indices: false + snapshot: + refresh_repo_uuid_on_restore: true + max_concurrent_operations: 1000 diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 4045fa10f..09cba56f9 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -15,6 +15,8 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} + + {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} @@ -36,6 +38,7 @@ {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} +{% from 'elasticsearch/config.map.jinja' import ESCONFIG with context %} vm.max_map_count: @@ -142,6 +145,14 @@ esyml: - group: 939 - template: jinja +esyml_test: + file.managed: + - name: /tmp/elasticsearch.yml + - source: salt://elasticsearch/files/elasticsearch.yml + - user: 930 + - group: 939 + - contents: {{ ESCONFIG | yaml }} + #sync templates to /opt/so/conf/elasticsearch/templates {% for TEMPLATE in TEMPLATES %} es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: From fd1e6323861edaafb7ef965f7de02d4aac2e3fb9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 25 Aug 2021 12:08:43 -0400 Subject: [PATCH 02/31] cleanup yaml --- salt/elasticsearch/defaults.yaml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 8fc244d6c..7c506a9d2 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -273,7 +273,7 @@ elasticsearch: - file - jar allowed_urls: [] - url: http: + url: "http:" action: auto_create_index: true search: @@ -514,8 +514,7 @@ elasticsearch: - RUN_AS_DENIED - RUN_AS_GRANTED - SECURITY_CONFIG_CHANGE - exclude: - [] + exclude: [] emit_node_host_address: false authc: password_hashing: @@ -618,8 +617,8 @@ elasticsearch: keystore: alias: slo_endpoint: - post: https: - redirect: https: + post: "https:" + redirect: "https:" defaults: nameid_format: urn:oasis:names:tc:SAML:2.0:nameid-format:transient authn_expiry: 5m @@ -632,10 +631,10 @@ elasticsearch: organization: display_name: name: - url: http: + url: "http:" sso_endpoint: - post: https: - redirect: https: + post: "https:" + redirect: "https:" entity_id: signing: keystore: @@ -658,7 +657,7 @@ elasticsearch: scheme: port: 0 whitelist: - - * + - "*" default_read_timeout: 10s max_response_size: 10mb autoscaling: From a27569f20b16e9fe4c4a1b63b2108a9e52bfa059 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 25 Aug 2021 12:32:17 -0400 Subject: [PATCH 03/31] remove source when contents provided --- salt/elasticsearch/init.sls | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 09cba56f9..49f9fc04c 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -148,7 +148,6 @@ esyml: esyml_test: file.managed: - name: /tmp/elasticsearch.yml - - source: salt://elasticsearch/files/elasticsearch.yml - user: 930 - group: 939 - contents: {{ ESCONFIG | yaml }} From f06ab8b77d3659e99bd9cd25470b58f99c3a0262 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 9 Sep 2021 08:55:36 -0400 Subject: [PATCH 04/31] testing defaults.yaml --- salt/elasticsearch/defaults.yaml | 426 +++++++++++++++---------------- salt/elasticsearch/init.sls | 3 +- 2 files changed, 213 insertions(+), 216 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 7c506a9d2..b3db5aa0c 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -56,17 +56,7 @@ elasticsearch: close: 45 delete: 365 - - - persistent: - cluster: - remote: - default-cluster-name: - seeds: - - 127.0.0.1:9300 - transient: - {} - defaults: + cluster_settings: cluster: max_voting_config_exclusions: 10 auto_shrink_voting_configuration: true @@ -86,9 +76,9 @@ elasticsearch: read_only: false remote: node: - attr: + attr: "" initial_connect_timeout: 30s - connect: true + #connect: true #DEPRECATION connections_per_cluster: 3 follower_lag: timeout: 90000ms @@ -99,16 +89,16 @@ elasticsearch: allocation: node_concurrent_incoming_recoveries: 2 include: - _tier: + _tier: "" node_initial_primaries_recoveries: 4 same_shard: host: false - total_shards_per_node: -1 + total_shards_per_node: "-1" require: - _tier: - shard_state: - reroute: - priority: NORMAL + _tier: "" + #shard_state: #DEPRECATION + #reroute: #DEPRECATION + #priority: NORMAL #DEPRECATION type: balanced disk: threshold_enabled: true @@ -119,7 +109,7 @@ elasticsearch: low: 95% enable_for_single_data_node: false flood_stage.frozen: 95% - include_relocations: true + #include_relocations: true #DEPRECATION reroute_interval: 60s awareness: attributes: [] @@ -133,7 +123,7 @@ elasticsearch: cluster_concurrent_rebalance: 2 node_concurrent_recoveries: 2 exclude: - _tier: + _tier: "" indices: tombstones: size: 500 @@ -148,7 +138,7 @@ elasticsearch: publish: timeout: 30000ms info_timeout: 10000ms - name: default-cluster-name + name: {{ grains.host }} # Will change if true cluster fault_detection: leader_check: interval: 1000ms @@ -158,10 +148,10 @@ elasticsearch: interval: 1000ms timeout: 10000ms retry_count: 3 - join: - timeout: 60000ms + #join: #DEPRECATION + #timeout: 60000ms #DEPRECATION max_shards_per_node: 1000 - initial_master_nodes: [] + #initial_master_nodes: [] # ERROR setting [cluster.initial_master_nodes] is not allowed when [discovery.type] is set to [single-node] snapshot: info: max_concurrent_fetches: 5 @@ -176,9 +166,9 @@ elasticsearch: level: INFO bootstrap: memory_lock: false - system_call_filter: true + #system_call_filter: true #DEPRECATION ctrlhandler: true - processors: 8 + #processors: 8 #DEPRECATION ingest: user_agent: cache_size: 1000 @@ -198,14 +188,14 @@ elasticsearch: - 0.0.0.0 tcp: reuse_address: true - keep_count: -1 - connect_timeout: 30s - keep_interval: -1 + keep_count: "-1" + #connect_timeout: 30s #DEPRECATION + keep_interval: "-1" no_delay: true keep_alive: true - receive_buffer_size: -1b - keep_idle: -1 - send_buffer_size: -1b + receive_buffer_size: "-1b" + keep_idle: "-1" + send_buffer_size: "-1b" bind_host: - 0.0.0.0 server: true @@ -215,15 +205,15 @@ elasticsearch: overhead: 2.0 publish_host: - 0.0.0.0 - pidfile: + pidfile: "" path: data: [] logs: /var/log/elasticsearch - shared_data: + shared_data: "" home: /usr/share/elasticsearch repo: [] search: - default_search_timeout: -1 + default_search_timeout: "-1" highlight: term_vector_multi_value: true default_allow_partial_results: true @@ -234,10 +224,10 @@ elasticsearch: keep_alive_interval: 1m remote: node: - attr: - initial_connect_timeout: 30s - connect: true - connections_per_cluster: 3 + attr: "" + #initial_connect_timeout: 30s #DEPRECATION + #connect: true #DEPRECATION + #connections_per_cluster: 3 #DEPRECATION default_keep_alive: 5m max_keep_alive: 24h aggs: @@ -262,9 +252,9 @@ elasticsearch: wait_for_metadata_timeout: 60s repositories: fs: - compress: false + #compress: false #DEPRECATION chunk_size: 9223372036854775807b - location: + location: "" url: supported_protocols: - http @@ -297,32 +287,32 @@ elasticsearch: queue_capacity: 1024 max_concurrent_policy_executions: 50 xpack: - flattened: - enabled: true + #flattened: #DEPRECATION + #enabled: true #DEPRECATION watcher: execution: scroll: size: 0 - timeout: + timeout: "" default_throttle_period: 5s internal: ops: bulk: - default_timeout: + default_timeout: "" index: - default_timeout: + default_timeout: "" search: - default_timeout: + default_timeout: "" thread_pool: queue_size: 1000 size: 40 index: rest: - direct_access: + direct_access: "" use_ilm_index_management: true - history: - cleaner_service: - enabled: true + #history: #DEPRECATION + #cleaner_service: #DEPRECATION + #enabled: true #DEPRECATION trigger: schedule: ticker: @@ -330,11 +320,11 @@ elasticsearch: enabled: true input: search: - default_timeout: + default_timeout: "" encrypt_sensitive_data: false transform: search: - default_timeout: + default_timeout: "" stop: timeout: 30s watch: @@ -347,15 +337,15 @@ elasticsearch: actions: 1 actions: bulk: - default_timeout: + default_timeout: "" index: - default_timeout: - eql: - enabled: true - data_frame: - enabled: true - ilm: - enabled: true + default_timeout: "" + #eql: #DEPRECATION + #enabled: true #DEPRECATION + #data_frame: #DEPRECATION + #enabled: true #DEPRECATION + #ilm: #DEPRECATION + #enabled: true #DEPRECATION monitoring: migration: decommission_alerts: false @@ -390,7 +380,7 @@ elasticsearch: elasticsearch: collection: enabled: true - enabled: true + #enabled: true #DEPRECATION graph: enabled: true searchable: @@ -410,16 +400,16 @@ elasticsearch: min_time_delta: 60s decay: interval: 60s - size.max_headroom: -1 + size.max_headroom: "-1" range_size: 16mb max_freq: 100 rollup: - enabled: true + #enabled: true #DEPRECATION task_thread_pool: - queue_size: -1 + queue_size: "-1" size: 1 - sql: - enabled: true + #sql: #DEPRECATION + #enabled: true #DEPRECATION searchable_snapshots: cache_fetch_async_thread_pool: core: 0 @@ -439,13 +429,13 @@ elasticsearch: - trial self_generated: type: basic - logstash: - enabled: true + #logstash: #DEPRECATION + #enabled: true #DEPRECATION notification: pagerduty: - default_account: + default_account: "" email: - default_account: + default_account: "" html: sanitization: allow: @@ -464,9 +454,9 @@ elasticsearch: enabled: true interval: 15s jira: - default_account: + default_account: "" slack: - default_account: + default_account: "" security: operator_privileges: enabled: false @@ -484,6 +474,10 @@ elasticsearch: enabled: true ssl: enabled: true + verification_mode: none + certificate_authorities: /usr/share/elasticsearch/config/ca.crt + key: /usr/share/elasticsearch/config/elasticsearch.key + certificate: /usr/share/elasticsearch/config/elasticsearch.crt ssl: diagnose: trust: true @@ -532,13 +526,13 @@ elasticsearch: ttl: 24h delete: interval: 24h - timeout: -1 + timeout: "-1" enabled: false hashing: algorithm: pbkdf2 anonymous: authz_exception: true - roles: + roles: - superuser username: anonymous_user run_as: @@ -553,7 +547,7 @@ elasticsearch: token: delete: interval: 30m - timeout: -1 + timeout: "-1" enabled: false thread_pool: queue_size: 1000 @@ -571,13 +565,17 @@ elasticsearch: enabled: true ssl: enabled: true + client_authentication: none + key: /usr/share/elasticsearch/config/elasticsearch.key + certificate: /usr/share/elasticsearch/config/elasticsearch.crt + certificate_authorities: /usr/share/elasticsearch/config/ca.crt automata: max_determinized_states: 100000 cache: size: 10000 ttl: 48h enabled: true - user: null + user: "" authz: store: privileges: @@ -585,10 +583,10 @@ elasticsearch: ttl: 24h max_size: 10000 roles: - index: - cache: - ttl: 20m - max_size: 10000 + #index: #DEPRECATION + #cache: #DEPRECATION + #ttl: 20m #DEPRECATION + #max_size: 10000 #DEPRECATION cache: max_size: 10000 negative_lookup_cache: @@ -598,9 +596,9 @@ elasticsearch: max_size_in_bytes: 104857600 transform: num_transform_failure_retries: 10 - enabled: true - vectors: - enabled: true + #enabled: true #DEPRECATION + #vectors: #DEPRECATION + #enabled: true #DEPRECATION ccr: enabled: true ccr_thread_pool: @@ -608,14 +606,14 @@ elasticsearch: size: 32 idp: privileges: - application: + application: "" cache: size: 100 ttl: 90m metadata: signing: keystore: - alias: + alias: "" slo_endpoint: post: "https:" redirect: "https:" @@ -625,20 +623,20 @@ elasticsearch: allowed_nameid_formats: - urn:oasis:names:tc:SAML:2.0:nameid-format:transient contact: - given_name: - email: - surname: + given_name: "" + email: "" + surname: "" organization: - display_name: - name: + display_name: "" + name: "" url: "http:" sso_endpoint: post: "https:" redirect: "https:" - entity_id: + entity_id: "" signing: keystore: - alias: + alias: "" sp: cache: size: 1000 @@ -646,15 +644,15 @@ elasticsearch: wildcard: path: wildcard_services.json enabled: false - slm: - enabled: true - enrich: - enabled: true + #slm: + #enabled: true #DEPRECATION + #enrich: #DEPRECATION + #enabled: true #DEPRECATION http: default_connection_timeout: 10s proxy: - host: - scheme: + host: "" + scheme: "" port: 0 whitelist: - "*" @@ -673,7 +671,7 @@ elasticsearch: inference_model: cache_size: 40% time_to_live: 5m - nightly_maintenance_requests_per_second: -1.0 + nightly_maintenance_requests_per_second: "-1.0" node_concurrent_job_allocations: 2 max_model_memory_limit: 0b enabled: false @@ -713,15 +711,15 @@ elasticsearch: medium: 30s thread_pool: force_merge: - queue_size: -1 + queue_size: "-1" size: 1 fetch_shard_started: core: 1 max: 16 keep_alive: 5m - listener: - queue_size: -1 - size: 4 + #listener: #DEPRECATION + #queue_size: "-1" #DEPRECATION + #size: 4 #DEPRECATION refresh: core: 1 max: 4 @@ -738,12 +736,12 @@ elasticsearch: max: 4 keep_alive: 5m search: - max_queue_size: 1000 + #max_queue_size: 1000 #DEPRECATION queue_size: 1000 size: 13 - auto_queue_frame_size: 2000 - target_response_time: 1s - min_queue_size: 1000 + #auto_queue_frame_size: 2000 #DEPRECATION + #target_response_time: 1s #DEPRECATION + #min_queue_size: 1000 #DEPRECATION fetch_shard_store: core: 1 max: 16 @@ -774,23 +772,23 @@ elasticsearch: max: 4 keep_alive: 5m search_throttled: - max_queue_size: 100 + #max_queue_size: 100 #DEPRECATION queue_size: 100 size: 1 - auto_queue_frame_size: 200 - target_response_time: 1s - min_queue_size: 100 + #auto_queue_frame_size: 200 #DEPRECATION + #target_response_time: 1s #DEPRECATION + #min_queue_size: 100 #DEPRECATION index: codec: default recovery: - type: + type: "" store: - type: + type: "" fs: fs_lock: native preload: [] snapshot: - uncached_chunk_size: -1b + uncached_chunk_size: "-1b" cache: excluded_file_types: [] monitor: @@ -821,21 +819,21 @@ elasticsearch: transport: tcp: reuse_address: true - keep_count: -1 - connect_timeout: 30s - keep_interval: -1 - compress: false - port: 9300-9400 + keep_count: "-1" + #connect_timeout: 30s #DEPRECATION + keep_interval: "-1" + #compress: false #DEPRECATION + #port: 9300-9400 #DEPRECATION no_delay: true keep_alive: true - receive_buffer_size: -1b - keep_idle: -1 - send_buffer_size: -1b + receive_buffer_size: "-1b" + keep_idle: "-1" + send_buffer_size: "-1b" bind_host: - 0.0.0.0 connect_timeout: 30s compress: false - ping_schedule: -1 + ping_schedule: "-1" connections_per_node: recovery: 2 state: 1 @@ -856,7 +854,7 @@ elasticsearch: port: 9300-9400 host: [] publish_port: 9300 - tcp_no_delay: true + #tcp_no_delay: true #DEPRECATION publish_host: {{ grains.host }} netty: receive_predictor_size: 64kb @@ -865,57 +863,57 @@ elasticsearch: receive_predictor_min: 64kb boss_count: 1 script: - allowed_contexts: [] - max_compilations_rate: 20000/1m - cache: - max_size: 100 - expire: 0ms + allowed_contexts: none # ERROR have to set to none - should be list + #max_compilations_rate: 20000/1m #DEPRECATION + #cache: #DEPRECATION + #max_size: 100 #DEPRECATION + #expire: 0ms #DEPRECATION painless: regex: enabled: limited limit-factor: 6 max_size_in_bytes: 65535 - allowed_types: [] + allowed_types: none # ERROR have to set to none - should be list disable_max_compilations_rate: false indexing_pressure: memory: limit: 10% node: - data: true - roles: - - data_frozen - - data_warm - - transform - - data - - remote_cluster_client - - data_cold - - data_content - - data_hot - - ingest - - master - max_local_storage_nodes: 1 + #data: true #DEPRECATION + # roles: + # - data_frozen + # - data_warm + # - transform ERROR + # - data + # - remote_cluster_client + # - data_cold + # - data_content + # - data_hot + # - ingest + # - master + #max_local_storage_nodes: 1 #DEPRECATION processors: 8 store: allow_mmap: true - ingest: true - master: true - pidfile: - transform: true - remote_cluster_client: true + #ingest: true #DEPRECATION + #master: true #DEPRECATION + pidfile: "" + #transform: true #DEPRECATION + #remote_cluster_client: true #DEPRECATION enable_lucene_segment_infos_trace: false - local_storage: true + #local_storage: true #DEPRECATION name: {{ grains.host }} id: seed: 0 - voting_only: false + #voting_only: false #DEPRECATION attr: - transform: - node: true + #transform: ERROR + # node: true ERROR xpack: - installed: true + installed: "" box_type: hot portsfile: false - ml: true + #ml: true #DEPRECATION indices: replication: retry_timeout: 60s @@ -927,7 +925,7 @@ elasticsearch: max_in_flight_updates: 10 memory: interval: 5s - max_index_buffer_size: -1 + max_index_buffer_size: "-1" shard_inactive_time: 5m index_buffer_size: 10% min_index_buffer_size: 48mb @@ -968,7 +966,7 @@ elasticsearch: requests: cache: size: 1% - expire: 0ms + expire: 1ms #0ms - ERROR when set to 0ms, set to 1ms and ERROR gone store: delete: shard: @@ -990,7 +988,7 @@ elasticsearch: master_timeout: 30s fielddata: cache: - size: -1b + size: "-1b" plugin: mandatory: [] slm: @@ -999,48 +997,48 @@ elasticsearch: retention_duration: 1h history_index_enabled: true discovery: - seed_hosts: [] + #seed_hosts: [] # ERROR - it is forbidden to set both [discovery.seed_hosts] and [discovery.zen.ping.unicast.hosts] unconfigured_bootstrap_timeout: 3s request_peers_timeout: 3000ms zen: - commit_timeout: 30s - no_master_block: write - join_retry_delay: 100ms - join_retry_attempts: 3 - ping: - unicast: - concurrent_connects: 10 - hosts: [] - hosts.resolve_timeout: 5s - master_election: - ignore_non_master_pings: false - wait_for_joins_timeout: 30000ms - send_leave_request: true + #commit_timeout: 30s #DEPRECATION + #no_master_block: write #DEPRECATION + #join_retry_delay: 100ms #DEPRECATION + #join_retry_attempts: 3 #DEPRECATION + #ping: + #unicast: + #concurrent_connects: 10 # ERROR forbidden to set both [discovery.seed_resolver.max_concurrent_resolvers] and [discovery.zen.ping.unicast.concurrent_connects] + #hosts: [] # ERROR - it is forbidden to set both [discovery.seed_hosts] and [discovery.zen.ping.unicast.hosts] + #hosts.resolve_timeout: 5s # ERROR forbidden to set both [discovery.seed_resolver.timeout] and [discovery.zen.ping.unicast.hosts.resolve_timeout] + #master_election: #DEPRECATION + #ignore_non_master_pings: false #DEPRECATION + #wait_for_joins_timeout: 30000ms #DEPRECATION + #send_leave_request: true #DEPRECATION ping_timeout: 3s - bwc_ping_timeout: 3s - join_timeout: 60000ms - publish_diff: - enable: true - publish: - max_pending_cluster_states: 25 - minimum_master_nodes: -1 - unsafe_rolling_upgrades_enabled: true - hosts_provider: [] - publish_timeout: 30s - fd: - connect_on_network_disconnect: false - ping_interval: 1s - ping_retries: 3 - register_connection_listener: true - ping_timeout: 30s - max_pings_from_another_master: 3 + #bwc_ping_timeout: 3s #DEPRECATION + #join_timeout: 60000ms #DEPRECATION + #publish_diff: + #enable: true #DEPRECATION + #publish: #DEPRECATION + #max_pending_cluster_states: 25 #DEPRECATION + #minimum_master_nodes: "-1" #DEPRECATION + #unsafe_rolling_upgrades_enabled: true #DEPRECATION + #hosts_provider: # ERROR forbidden to set both [discovery.seed_providers] and [discovery.zen.hosts_provider] has to be commented out + #publish_timeout: 30s #DEPRECATION + #fd: #DEPRECATION + #connect_on_network_disconnect: false #DEPRECATION + #ping_interval: 1s #DEPRECATION + #ping_retries: 3 #DEPRECATION + #register_connection_listener: true #DEPRECATION + #ping_timeout: 30s #DEPRECATION + #max_pings_from_another_master: 3 #DEPRECATION initial_state_timeout: 30s cluster_formation_warning_timeout: 10000ms - seed_providers: [] - type: single-node - seed_resolver: - max_concurrent_resolvers: 10 - timeout: 5s + #seed_providers: # ERROR forbidden to set both [discovery.seed_providers] and [discovery.zen.hosts_provider] has to be commented out + type: zen # ERROR java.lang.IllegalArgumentException: node with [discovery.type] set to [single-node] must be master-eligible # test turning off + #seed_resolver: + #max_concurrent_resolvers: 10 # ERROR forbidden to set both [discovery.seed_resolver.max_concurrent_resolvers] and [discovery.zen.ping.unicast.concurrent_connects] + #timeout: 5s # forbidden to set both [discovery.seed_resolver.timeout] and [discovery.zen.ping.unicast.hosts.resolve_timeout] find_peers_interval: 1000ms probe: connect_timeout: 30s @@ -1048,7 +1046,7 @@ elasticsearch: http: cors: max-age: 1728000 - allow-origin: + allow-origin: "" allow-headers: X-Requested-With,Content-Type,Content-Length allow-credentials: false allow-methods: OPTIONS,HEAD,GET,POST,PUT,DELETE @@ -1060,10 +1058,10 @@ elasticsearch: pipelining: max_events: 10000 type.default: netty4 - content_type: - required: true + #content_type: #DEPRECATION + #required: true #DEPRECATION host: [] - publish_port: -1 + publish_port: "-1" read_timeout: 0ms max_content_length: 100mb netty: @@ -1072,40 +1070,40 @@ elasticsearch: worker_count: 0 tcp: reuse_address: true - keep_count: -1 - keep_interval: -1 + keep_count: "-1" + keep_interval: "-1" no_delay: true keep_alive: true - receive_buffer_size: -1b - keep_idle: -1 - send_buffer_size: -1b + receive_buffer_size: "-1b" + keep_idle: "-1" + send_buffer_size: "-1b" bind_host: [] client_stats: enabled: true reset_cookies: false - max_warning_header_count: -1 + max_warning_header_count: "-1" tracer: include: [] exclude: [] - max_warning_header_size: -1b + max_warning_header_size: "-1b" detailed_errors: enabled: true port: 9200-9300 max_header_size: 8kb - tcp_no_delay: true + #tcp_no_delay: true #DEPRECATION compression: false publish_host: [] gateway: - recover_after_master_nodes: 0 - expected_nodes: -1 - recover_after_data_nodes: -1 - expected_data_nodes: -1 + #recover_after_master_nodes: 0 #DEPRECATION + #expected_nodes: "-1" #DEPRECATION + recover_after_data_nodes: "-1" + expected_data_nodes: "-1" write_dangling_indices_info: true slow_write_logging_threshold: 10s recover_after_time: 0ms - expected_master_nodes: -1 - recover_after_nodes: -1 - auto_import_dangling_indices: false + #expected_master_nodes: "-1" #DEPRECATION + #recover_after_nodes: "-1" #DEPRECATION + #auto_import_dangling_indices: false #DEPRECATION snapshot: refresh_repo_uuid_on_restore: true max_concurrent_operations: 1000 diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 49f9fc04c..2deeef55c 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -140,10 +140,9 @@ eslog4jfile: esyml: file.managed: - name: /opt/so/conf/elasticsearch/elasticsearch.yml - - source: salt://elasticsearch/files/elasticsearch.yml + - contents: {{ ESCONFIG.elasticsearch.cluster_settings | yaml }} - user: 930 - group: 939 - - template: jinja esyml_test: file.managed: From 93f2cd75a4c540e289305a8c712783ebb498b5aa Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 9 Sep 2021 10:19:46 -0400 Subject: [PATCH 05/31] add the jinja template --- salt/elasticsearch/files/elasticsearch.yaml.jinja | 1 + salt/elasticsearch/init.sls | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 salt/elasticsearch/files/elasticsearch.yaml.jinja diff --git a/salt/elasticsearch/files/elasticsearch.yaml.jinja b/salt/elasticsearch/files/elasticsearch.yaml.jinja new file mode 100644 index 000000000..caf13173f --- /dev/null +++ b/salt/elasticsearch/files/elasticsearch.yaml.jinja @@ -0,0 +1 @@ +{{ ESCONFIG | yaml(False) }} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 2deeef55c..7e59f5c43 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -140,9 +140,13 @@ eslog4jfile: esyml: file.managed: - name: /opt/so/conf/elasticsearch/elasticsearch.yml - - contents: {{ ESCONFIG.elasticsearch.cluster_settings | yaml }} + - source: salt://elasticsearch/files/elasticsearch.yaml.jinja - user: 930 - group: 939 + - defaults: + ESCONFIG: {{ ESCONFIG.elasticsearch.cluster_settings }} + - template: jinja + esyml_test: file.managed: From 0ae09cc6309ca237e4febf2f1425c1fca0405d92 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 13 Sep 2021 09:49:56 -0400 Subject: [PATCH 06/31] config changes --- salt/elasticsearch/defaults.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index b3db5aa0c..d84349963 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -863,8 +863,8 @@ elasticsearch: receive_predictor_min: 64kb boss_count: 1 script: - allowed_contexts: none # ERROR have to set to none - should be list - #max_compilations_rate: 20000/1m #DEPRECATION + #allowed_contexts: none # ERROR have to set to none - should be list + max_compilations_rate: 20000/1m #DEPRECATION #cache: #DEPRECATION #max_size: 100 #DEPRECATION #expire: 0ms #DEPRECATION @@ -873,7 +873,7 @@ elasticsearch: enabled: limited limit-factor: 6 max_size_in_bytes: 65535 - allowed_types: none # ERROR have to set to none - should be list + #allowed_types: none # ERROR have to set to none - should be list disable_max_compilations_rate: false indexing_pressure: memory: From f8ab0ac8a908359432a2a95cf7649c604ad120b1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 13 Sep 2021 15:04:39 -0400 Subject: [PATCH 07/31] config changes --- salt/elasticsearch/defaults.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index d84349963..d7f21b88e 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -532,9 +532,9 @@ elasticsearch: algorithm: pbkdf2 anonymous: authz_exception: true - roles: - - superuser - username: anonymous_user + #roles: + # - superuser + username: _anonymous run_as: enabled: true reserved_realm: @@ -966,7 +966,7 @@ elasticsearch: requests: cache: size: 1% - expire: 1ms #0ms - ERROR when set to 0ms, set to 1ms and ERROR gone + #expire: 1ms #0ms - ERROR when set to 0ms, set to 1ms and ERROR gone store: delete: shard: From 614a6dc9fe46db0f75da51408e2703c14c426abd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 14 Sep 2021 13:56:43 -0400 Subject: [PATCH 08/31] Update manager.sls --- pillar/elasticsearch/manager.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pillar/elasticsearch/manager.sls b/pillar/elasticsearch/manager.sls index 9ff97de5b..84ff89a23 100644 --- a/pillar/elasticsearch/manager.sls +++ b/pillar/elasticsearch/manager.sls @@ -1,7 +1,7 @@ elasticsearch: templates: - so/so-beats-template.json.jinja - - so/so-common-template.json + - so/so-common-template.json.jinja - so/so-firewall-template.json.jinja - so/so-flow-template.json.jinja - so/so-ids-template.json.jinja From 2cc25587d98f2cc469589aab540d3b62efae59a1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 14 Sep 2021 13:57:04 -0400 Subject: [PATCH 09/31] Update eval.sls --- pillar/elasticsearch/eval.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pillar/elasticsearch/eval.sls b/pillar/elasticsearch/eval.sls index 2dbb08f59..84ff89a23 100644 --- a/pillar/elasticsearch/eval.sls +++ b/pillar/elasticsearch/eval.sls @@ -1,7 +1,7 @@ elasticsearch: templates: - so/so-beats-template.json.jinja - - so/so-common-template.json + - so/so-common-template.json.jinja - so/so-firewall-template.json.jinja - so/so-flow-template.json.jinja - so/so-ids-template.json.jinja @@ -10,4 +10,4 @@ elasticsearch: - so/so-ossec-template.json.jinja - so/so-strelka-template.json.jinja - so/so-syslog-template.json.jinja - - so/so-zeek-template.json.jinja \ No newline at end of file + - so/so-zeek-template.json.jinja From 6ae2fba71fa50a5e1cf3fd578562bcfbfa7dff36 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 14 Sep 2021 13:57:26 -0400 Subject: [PATCH 10/31] Update search.sls --- pillar/elasticsearch/search.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pillar/elasticsearch/search.sls b/pillar/elasticsearch/search.sls index 9ff97de5b..84ff89a23 100644 --- a/pillar/elasticsearch/search.sls +++ b/pillar/elasticsearch/search.sls @@ -1,7 +1,7 @@ elasticsearch: templates: - so/so-beats-template.json.jinja - - so/so-common-template.json + - so/so-common-template.json.jinja - so/so-firewall-template.json.jinja - so/so-flow-template.json.jinja - so/so-ids-template.json.jinja From 2ffb723bbd77c3669687b13a42d8029e8b2d98a7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 14 Sep 2021 13:58:45 -0400 Subject: [PATCH 11/31] Rename so-common-template.json to so-common-template.json.jinja --- .../so/{so-common-template.json => so-common-template.json.jinja} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename salt/elasticsearch/templates/so/{so-common-template.json => so-common-template.json.jinja} (100%) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json.jinja similarity index 100% rename from salt/elasticsearch/templates/so/so-common-template.json rename to salt/elasticsearch/templates/so/so-common-template.json.jinja From 551dba955ca6cdbfbca6eff9a5aea7483a21eedb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 09:20:33 -0400 Subject: [PATCH 12/31] set roles empty list --- salt/elasticsearch/defaults.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index d7f21b88e..bfe0e150b 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -532,8 +532,7 @@ elasticsearch: algorithm: pbkdf2 anonymous: authz_exception: true - #roles: - # - superuser + roles: [] username: _anonymous run_as: enabled: true From e3e2e1d851760c9e774b5c735d1a6b9a32a5da6f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 13:09:04 -0400 Subject: [PATCH 13/31] logic for truecluster to map file --- salt/elasticsearch/config.map.jinja | 30 +++++++++++++++++++++++++++-- salt/elasticsearch/defaults.yaml | 1 - 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index e6860e699..638747ec2 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,5 +1,31 @@ -{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG %} +# soup needs +# elasticsearch:esclustername pillar move to elasticsearch:cluster_settings:cluster:name +# move elasticsearch:true_cluster_name to elasticsearch:cluster_settings:cluster:name if true_cluster enabled +# elasticsearch:node_route_type moved + +{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} {% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} - {% do ESCONFIG.elasticsearch.defaults.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} + {% do ESCONFIG.elasticsearch.cluster_settings.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} {% endif %} + +{% if salt['pillar.get']('elasticsearch:true_cluster', False) %} + {% if grains.id.split('_') | last in ['manager','managersearch'] %} + {% if salt['pillar.get']('nodestab', {}) %} + {% set ESCONFIG.elasticsearch.cluster_settings.node.roles = ['master', 'data', 'remote_cluster_client'] %} +node.roles: [ master, data, remote_cluster_client ] + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} + {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} + {% endfor %} + {% endif %} + {% if grains.id.split('_') | last == 'managersearch' %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} + {% endif %} + {% else %} + {% set ESCONFIG.elasticsearch.cluster_settings.node.roles = ['data', 'ingest'] %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} + {% endif %} +{% endif %} + diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index bfe0e150b..817f63b0f 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,7 +1,6 @@ elasticsearch: es_port: 9200 esheap: 4049m - esclustername: default-cluster-name log_size_limit: 95 #used for curator index_settings: From 1a03853a7cbcd9c86964dc599d3e43313788bd4b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 13:38:29 -0400 Subject: [PATCH 14/31] fix extend --- salt/elasticsearch/config.map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 638747ec2..3f39a9aa3 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -12,7 +12,7 @@ {% if salt['pillar.get']('elasticsearch:true_cluster', False) %} {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} - {% set ESCONFIG.elasticsearch.cluster_settings.node.roles = ['master', 'data', 'remote_cluster_client'] %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.roles.extend(['master', 'data', 'remote_cluster_client']) %} node.roles: [ master, data, remote_cluster_client ] {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} @@ -23,7 +23,7 @@ node.roles: [ master, data, remote_cluster_client ] {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} {% endif %} {% else %} - {% set ESCONFIG.elasticsearch.cluster_settings.node.roles = ['data', 'ingest'] %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.roles.extend(['data', 'ingest']) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} {% endif %} From 1614b70853ea4a4c7820a4b16fb4639eea5475b1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 13:45:43 -0400 Subject: [PATCH 15/31] update cluster name if true cluster --- salt/elasticsearch/config.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 3f39a9aa3..2c62a0a71 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -10,10 +10,10 @@ {% endif %} {% if salt['pillar.get']('elasticsearch:true_cluster', False) %} + {% do ESCONFIG.elasticsearch.cluster_settings.cluster.update({'name': salt['pillar.get']('elasticsearch:true_cluster_name')}) %} {# this is temporary #} {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.roles.extend(['master', 'data', 'remote_cluster_client']) %} -node.roles: [ master, data, remote_cluster_client ] {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} From c2138343169469d7b19e4d0aa144d370cada4f82 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 15:24:40 -0400 Subject: [PATCH 16/31] update the dict --- salt/elasticsearch/config.map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 2c62a0a71..3d1e7b12a 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -13,7 +13,7 @@ {% do ESCONFIG.elasticsearch.cluster_settings.cluster.update({'name': salt['pillar.get']('elasticsearch:true_cluster_name')}) %} {# this is temporary #} {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.roles.extend(['master', 'data', 'remote_cluster_client']) %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.update('roles': ['master', 'data', 'remote_cluster_client']) %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} @@ -23,7 +23,7 @@ {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} {% endif %} {% else %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.roles.extend(['data', 'ingest']) %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.update('roles': ['data', 'ingest']) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} {% endif %} From e0dc62b6e9047844bcd43e9af58630d3591aa74d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 15:43:47 -0400 Subject: [PATCH 17/31] fix dict update --- salt/elasticsearch/config.map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 3d1e7b12a..b89e04bfe 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -13,7 +13,7 @@ {% do ESCONFIG.elasticsearch.cluster_settings.cluster.update({'name': salt['pillar.get']('elasticsearch:true_cluster_name')}) %} {# this is temporary #} {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.update('roles': ['master', 'data', 'remote_cluster_client']) %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} @@ -23,7 +23,7 @@ {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} {% endif %} {% else %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.update('roles': ['data', 'ingest']) %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['data', 'ingest']}) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} {% endif %} From a43b37f234a3f2a64a4c85b410699bee6f726d46 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 15:49:18 -0400 Subject: [PATCH 18/31] fix dict update --- salt/elasticsearch/config.map.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index b89e04bfe..45ce5ed19 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -14,9 +14,9 @@ {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update.({'seed_hosts': grains.master}) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': SN.split('_')|first}) %} {% endfor %} {% endif %} {% if grains.id.split('_') | last == 'managersearch' %} @@ -25,7 +25,7 @@ {% else %} {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['data', 'ingest']}) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(grains.master) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': grains.master}) %} {% endif %} {% endif %} From 4cf91f6c86942d5693a0a7f3a9b67a655d9ec247 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 15:51:00 -0400 Subject: [PATCH 19/31] fix dict update --- salt/elasticsearch/config.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 45ce5ed19..beb811e4b 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -14,7 +14,7 @@ {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update.({'seed_hosts': grains.master}) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': grains.master}) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': SN.split('_')|first}) %} {% endfor %} From 3bf9685df8bd621f8cafb424a6971d40cad08381 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 17:00:16 -0400 Subject: [PATCH 20/31] fix seed_hosts append --- salt/elasticsearch/config.map.jinja | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index beb811e4b..da40f578e 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -16,7 +16,7 @@ {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': grains.master}) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': SN.split('_')|first}) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} {% endfor %} {% endif %} {% if grains.id.split('_') | last == 'managersearch' %} @@ -28,4 +28,3 @@ {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': grains.master}) %} {% endif %} {% endif %} - From 782b01e76f5bf4cd2bb2e96c28022501f77f75b3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 Sep 2021 17:07:52 -0400 Subject: [PATCH 21/31] seed_hosts to list --- salt/elasticsearch/config.map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index da40f578e..82808971a 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -14,7 +14,7 @@ {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': grains.master}) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': [grains.master]}) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} {% endfor %} @@ -25,6 +25,6 @@ {% else %} {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['data', 'ingest']}) %} {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': grains.master}) %} + {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': [grains.master]}) %} {% endif %} {% endif %} From 889d235c4577af4820cde5115f73888a0b3e037a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 16 Sep 2021 09:15:24 -0400 Subject: [PATCH 22/31] no box type more manager in true cluster --- salt/elasticsearch/config.map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 82808971a..5f29dba3b 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -19,8 +19,8 @@ {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} {% endfor %} {% endif %} - {% if grains.id.split('_') | last == 'managersearch' %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} + {% if grains.id.split('_') | last == 'manager' %} + {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': ''}) %} {% endif %} {% else %} {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['data', 'ingest']}) %} From 5526a2bc3affd776df03afd969f7cc5b163e1664 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 16 Sep 2021 15:32:08 -0400 Subject: [PATCH 23/31] reduce defaults.yaml --- salt/elasticsearch/config.map.jinja | 22 +- salt/elasticsearch/defaults.yaml | 1102 +-------------------------- 2 files changed, 38 insertions(+), 1086 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 5f29dba3b..81ff4d157 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,30 +1,30 @@ # soup needs -# elasticsearch:esclustername pillar move to elasticsearch:cluster_settings:cluster:name -# move elasticsearch:true_cluster_name to elasticsearch:cluster_settings:cluster:name if true_cluster enabled +# elasticsearch:esclustername pillar move to elasticsearch:config:cluster:name +# move elasticsearch:true_cluster_name to elasticsearch:config:cluster:name if true_cluster enabled # elasticsearch:node_route_type moved {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} {% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} - {% do ESCONFIG.elasticsearch.cluster_settings.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} + {% do ESCONFIG.elasticsearch.config.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} {% endif %} {% if salt['pillar.get']('elasticsearch:true_cluster', False) %} - {% do ESCONFIG.elasticsearch.cluster_settings.cluster.update({'name': salt['pillar.get']('elasticsearch:true_cluster_name')}) %} {# this is temporary #} + {% do ESCONFIG.elasticsearch.config.cluster.update({'name': salt['pillar.get']('elasticsearch:true_cluster_name')}) %} {# this is temporary #} {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': [grains.master]}) %} + {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} + {% do ESCONFIG.elasticsearch.config.discovery.update({'seed_hosts': [grains.master]}) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.seed_hosts.append(SN.split('_')|first) %} + {% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(SN.split('_')|first) %} {% endfor %} {% endif %} {% if grains.id.split('_') | last == 'manager' %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': ''}) %} + {% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': ''}) %} {% endif %} {% else %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.update({'roles': ['data', 'ingest']}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.node.attr.update({'box_type': 'hot'}) %} - {% do ESCONFIG.elasticsearch.cluster_settings.discovery.update({'seed_hosts': [grains.master]}) %} + {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['data', 'ingest']}) %} + {% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': 'hot'}) %} + {% do ESCONFIG.elasticsearch.config.discovery.update({'seed_hosts': [grains.master]}) %} {% endif %} {% endif %} diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 817f63b0f..12d5b27fa 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,1107 +1,59 @@ elasticsearch: - es_port: 9200 - esheap: 4049m - log_size_limit: 95 #used for curator - - index_settings: - so-beats: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-firewall: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-flow: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-ids: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-import: - shards: 1 - warm: 7 - close: 73000 - delete: 73001 - so-osquery: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-ossec: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-strelka: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-syslog: - shards: 1 - warm: 7 - close: 30 - delete: 365 - so-zeek: - shards: 5 - warm: 7 - close: 45 - delete: 365 - - cluster_settings: + config: + node: + name: {{ grains.host }} + attr: + box_type: hot cluster: - max_voting_config_exclusions: 10 - auto_shrink_voting_configuration: true - election: - duration: 500ms - initial_timeout: 100ms - max_timeout: 10s - back_off_time: 100ms - strategy: supports_voting_only - no_master_block: write - persistent_tasks: - allocation: - enable: all - recheck_interval: 30s - blocks: - read_only_allow_delete: false - read_only: false - remote: - node: - attr: "" - initial_connect_timeout: 30s - #connect: true #DEPRECATION - connections_per_cluster: 3 - follower_lag: - timeout: 90000ms + name: {{ grains.host }} routing: - use_adaptive_replica_selection: true - rebalance: - enable: all allocation: - node_concurrent_incoming_recoveries: 2 - include: - _tier: "" - node_initial_primaries_recoveries: 4 - same_shard: - host: false - total_shards_per_node: "-1" - require: - _tier: "" - #shard_state: #DEPRECATION - #reroute: #DEPRECATION - #priority: NORMAL #DEPRECATION - type: balanced disk: threshold_enabled: true watermark: - flood_stage.frozen.max_headroom: 20GB - flood_stage: 98% - high: 98% low: 95% - enable_for_single_data_node: false - flood_stage.frozen: 95% - #include_relocations: true #DEPRECATION - reroute_interval: 60s - awareness: - attributes: [] - balance: - index: 0.55 - threshold: 1.0 - shard: 0.45 - enable: all - node_concurrent_outgoing_recoveries: 2 - allow_rebalance: indices_all_active - cluster_concurrent_rebalance: 2 - node_concurrent_recoveries: 2 - exclude: - _tier: "" - indices: - tombstones: - size: 500 - close: - enable: true - max_shards_per_node.frozen: 3000 - nodes: - reconnect_interval: 10s - service: - slow_master_task_logging_threshold: 10s - slow_task_logging_threshold: 30s - publish: - timeout: 30000ms - info_timeout: 10000ms - name: {{ grains.host }} # Will change if true cluster - fault_detection: - leader_check: - interval: 1000ms - timeout: 10000ms - retry_count: 3 - follower_check: - interval: 1000ms - timeout: 10000ms - retry_count: 3 - #join: #DEPRECATION - #timeout: 60000ms #DEPRECATION - max_shards_per_node: 1000 - #initial_master_nodes: [] # ERROR setting [cluster.initial_master_nodes] is not allowed when [discovery.type] is set to [single-node] - snapshot: - info: - max_concurrent_fetches: 5 - info: - update: - interval: 30s - timeout: 15s - stack: - templates: - enabled: true - logger: - level: INFO - bootstrap: - memory_lock: false - #system_call_filter: true #DEPRECATION - ctrlhandler: true - #processors: 8 #DEPRECATION - ingest: - user_agent: - cache_size: 1000 - geoip: - cache_size: 1000 - downloader: - enabled: false - endpoint: https://geoip.elastic.co/v1/database - poll: - interval: 3d - grok: - watchdog: - max_execution_time: 1s - interval: 1s + high: 98% + flood_stage: 98% network: - host: - - 0.0.0.0 - tcp: - reuse_address: true - keep_count: "-1" - #connect_timeout: 30s #DEPRECATION - keep_interval: "-1" - no_delay: true - keep_alive: true - receive_buffer_size: "-1b" - keep_idle: "-1" - send_buffer_size: "-1b" - bind_host: - - 0.0.0.0 - server: true - breaker: - inflight_requests: - limit: 100% - overhead: 2.0 - publish_host: - - 0.0.0.0 - pidfile: "" + host: 0.0.0.0 path: - data: [] logs: /var/log/elasticsearch - shared_data: "" - home: /usr/share/elasticsearch - repo: [] - search: - default_search_timeout: "-1" - highlight: - term_vector_multi_value: true - default_allow_partial_results: true - max_open_scroll_context: 500 - max_buckets: 65536 - low_level_cancellation: true - allow_expensive_queries: true - keep_alive_interval: 1m - remote: - node: - attr: "" - #initial_connect_timeout: 30s #DEPRECATION - #connect: true #DEPRECATION - #connections_per_cluster: 3 #DEPRECATION - default_keep_alive: 5m - max_keep_alive: 24h - aggs: - rewrite_to_filter_by_filter: true - security: - manager: - filter_bad_defaults: true - transform: - task_thread_pool: - queue_size: 4 - size: 4 - ccr: - wait_for_metadata_timeout: 60s - indices: - recovery: - recovery_activity_timeout: 60s - chunk_size: 1mb - internal_action_timeout: 60s - max_bytes_per_sec: 40mb - max_concurrent_file_chunks: 5 - auto_follow: - wait_for_metadata_timeout: 60s - repositories: - fs: - #compress: false #DEPRECATION - chunk_size: 9223372036854775807b - location: "" - url: - supported_protocols: - - http - - https - - ftp - - file - - jar - allowed_urls: [] - url: "http:" action: - auto_create_index: true - search: - shard_count: - limit: 9223372036854775807 destructive_requires_name: true - client: - type: node - transport: - ignore_cluster_name: false - nodes_sampler_interval: 5s - sniff: false - ping_timeout: 5s - enrich: - max_force_merge_attempts: 3 - cleanup_period: 15m - fetch_size: 10000 - coordinator_proxy: - max_concurrent_requests: 8 - max_lookups_per_request: 128 - queue_capacity: 1024 - max_concurrent_policy_executions: 50 + transport: + bind_host: 0.0.0.0 + publish_host: {{ grains.host }} + publish_port: 9300 xpack: - #flattened: #DEPRECATION - #enabled: true #DEPRECATION - watcher: - execution: - scroll: - size: 0 - timeout: "" - default_throttle_period: 5s - internal: - ops: - bulk: - default_timeout: "" - index: - default_timeout: "" - search: - default_timeout: "" - thread_pool: - queue_size: 1000 - size: 40 - index: - rest: - direct_access: "" - use_ilm_index_management: true - #history: #DEPRECATION - #cleaner_service: #DEPRECATION - #enabled: true #DEPRECATION - trigger: - schedule: - ticker: - tick_interval: 500ms - enabled: true - input: - search: - default_timeout: "" - encrypt_sensitive_data: false - transform: - search: - default_timeout: "" - stop: - timeout: 30s - watch: - scroll: - size: 0 - bulk: - concurrent_requests: 0 - flush_interval: 1s - size: 1mb - actions: 1 - actions: - bulk: - default_timeout: "" - index: - default_timeout: "" - #eql: #DEPRECATION - #enabled: true #DEPRECATION - #data_frame: #DEPRECATION - #enabled: true #DEPRECATION - #ilm: #DEPRECATION - #enabled: true #DEPRECATION - monitoring: - migration: - decommission_alerts: false - collection: - cluster: - stats: - timeout: 10s - node: - stats: - timeout: 10s - indices: [] - ccr: - stats: - timeout: 10s - enrich: - stats: - timeout: 10s - index: - stats: - timeout: 10s - recovery: - active_only: false - timeout: 10s - interval: 10s - enabled: false - ml: - job: - stats: - timeout: 10s - history: - duration: 168h - elasticsearch: - collection: - enabled: true - #enabled: true #DEPRECATION - graph: - enabled: true - searchable: - snapshot: - allocate_on_rolling_restart: false - cache: - range_size: 32mb - sync: - max_files: 10000 - interval: 60s - shutdown_timeout: 10s - recovery_range_size: 128kb - shared_cache: - recovery_range_size: 128kb - region_size: 16mb - size: 0 - min_time_delta: 60s - decay: - interval: 60s - size.max_headroom: "-1" - range_size: 16mb - max_freq: 100 - rollup: - #enabled: true #DEPRECATION - task_thread_pool: - queue_size: "-1" - size: 1 - #sql: #DEPRECATION - #enabled: true #DEPRECATION - searchable_snapshots: - cache_fetch_async_thread_pool: - core: 0 - max: 24 - keep_alive: 30s - cache_prewarming_thread_pool: - core: 0 - max: 16 - keep_alive: 30s - license: - upload: - types: - - standard - - gold - - platinum - - enterprise - - trial - self_generated: - type: basic - #logstash: #DEPRECATION - #enabled: true #DEPRECATION - notification: - pagerduty: - default_account: "" - email: - default_account: "" - html: - sanitization: - allow: - - body - - head - - _tables - - _links - - _blocks - - _formatting - - img:embedded - disallow: [] - enabled: true - reporting: - retries: 40 - warning: - enabled: true - interval: 15s - jira: - default_account: "" - slack: - default_account: "" security: - operator_privileges: - enabled: false - dls_fls: - enabled: true - dls: - bitset: - cache: - size: 10% - ttl: 2h - transport: - filter: - allow: [] - deny: [] - enabled: true - ssl: - enabled: true - verification_mode: none - certificate_authorities: /usr/share/elasticsearch/config/ca.crt - key: /usr/share/elasticsearch/config/elasticsearch.key - certificate: /usr/share/elasticsearch/config/elasticsearch.crt - ssl: - diagnose: - trust: true - enabled: true - crypto: - thread_pool: - queue_size: 1000 - size: 4 - filter: - always_allow_bound_address: true - encryption: - algorithm: AES/CTR/NoPadding - audit: - enabled: false - logfile: - emit_node_id: true - emit_node_host_name: false - emit_node_name: false - events: - emit_request_body: false - include: - - ACCESS_DENIED - - ACCESS_GRANTED - - ANONYMOUS_ACCESS_DENIED - - AUTHENTICATION_FAILED - - CONNECTION_DENIED - - TAMPERED_REQUEST - - RUN_AS_DENIED - - RUN_AS_GRANTED - - SECURITY_CONFIG_CHANGE - exclude: [] - emit_node_host_address: false authc: - password_hashing: - algorithm: bcrypt - success_cache: - size: 10000 - enabled: true - expire_after_access: 1h - api_key: - doc_cache: - ttl: 5m - cache: - hash_algo: ssha256 - max_keys: 10000 - ttl: 24h - delete: - interval: 24h - timeout: "-1" - enabled: false - hashing: - algorithm: pbkdf2 anonymous: authz_exception: true roles: [] username: _anonymous - run_as: + transport: + ssl: enabled: true - reserved_realm: - enabled: true - service_token: - cache: - hash_algo: ssha256 - max_tokens: 100000 - ttl: 20m - token: - delete: - interval: 30m - timeout: "-1" - enabled: false - thread_pool: - queue_size: 1000 - size: 1 - timeout: 20m - fips_mode: - enabled: false - encryption_key: - length: 128 - algorithm: AES + verification_mode: none + key: /usr/share/elasticsearch/config/elasticsearch.key + certificate: /usr/share/elasticsearch/config/elasticsearch.crt + certificate_authorities: + - /usr/share/elasticsearch/config/ca.crt http: - filter: - allow: [] - deny: [] - enabled: true ssl: enabled: true client_authentication: none key: /usr/share/elasticsearch/config/elasticsearch.key certificate: /usr/share/elasticsearch/config/elasticsearch.crt - certificate_authorities: /usr/share/elasticsearch/config/ca.crt - automata: - max_determinized_states: 100000 - cache: - size: 10000 - ttl: 48h - enabled: true - user: "" - authz: - store: - privileges: - cache: - ttl: 24h - max_size: 10000 - roles: - #index: #DEPRECATION - #cache: #DEPRECATION - #ttl: 20m #DEPRECATION - #max_size: 10000 #DEPRECATION - cache: - max_size: 10000 - negative_lookup_cache: - max_size: 10000 - field_permissions: - cache: - max_size_in_bytes: 104857600 - transform: - num_transform_failure_retries: 10 - #enabled: true #DEPRECATION - #vectors: #DEPRECATION - #enabled: true #DEPRECATION - ccr: - enabled: true - ccr_thread_pool: - queue_size: 100 - size: 32 - idp: - privileges: - application: "" - cache: - size: 100 - ttl: 90m - metadata: - signing: - keystore: - alias: "" - slo_endpoint: - post: "https:" - redirect: "https:" - defaults: - nameid_format: urn:oasis:names:tc:SAML:2.0:nameid-format:transient - authn_expiry: 5m - allowed_nameid_formats: - - urn:oasis:names:tc:SAML:2.0:nameid-format:transient - contact: - given_name: "" - email: "" - surname: "" - organization: - display_name: "" - name: "" - url: "http:" - sso_endpoint: - post: "https:" - redirect: "https:" - entity_id: "" - signing: - keystore: - alias: "" - sp: - cache: - size: 1000 - ttl: 60m - wildcard: - path: wildcard_services.json - enabled: false - #slm: - #enabled: true #DEPRECATION - #enrich: #DEPRECATION - #enabled: true #DEPRECATION - http: - default_connection_timeout: 10s - proxy: - host: "" - scheme: "" - port: 0 - whitelist: - - "*" - default_read_timeout: 10s - max_response_size: 10mb - autoscaling: - memory: - monitor: - timeout: 15s - ml: - max_anomaly_records: 500 - enable_config_migration: true - max_open_jobs: 512 - min_disk_space_off_heap: 5gb - use_auto_machine_memory_percent: false - inference_model: - cache_size: 40% - time_to_live: 5m - nightly_maintenance_requests_per_second: "-1.0" - node_concurrent_job_allocations: 2 - max_model_memory_limit: 0b - enabled: false - max_lazy_ml_nodes: 0 - max_ml_node_size: 0b - max_machine_memory_percent: 30 - persist_results_max_retries: 20 - autodetect_process: true - max_inference_processors: 50 - process_connect_timeout: 10s - rest: - action: - multi: - allow_explicit_index: true - cache: - recycler: - page: - limit: - heap: 10% - type: CONCURRENT - weight: - longs: 1.0 - ints: 1.0 - bytes: 1.0 - objects: 0.1 - async_search: - index_cleanup_interval: 1h - reindex: - remote: - whitelist: [] - resource: - reload: - enabled: true - interval: - low: 60s - high: 5s - medium: 30s - thread_pool: - force_merge: - queue_size: "-1" - size: 1 - fetch_shard_started: - core: 1 - max: 16 - keep_alive: 5m - #listener: #DEPRECATION - #queue_size: "-1" #DEPRECATION - #size: 4 #DEPRECATION - refresh: - core: 1 - max: 4 - keep_alive: 5m - system_write: - queue_size: 1000 - size: 4 - generic: - core: 4 - max: 128 - keep_alive: 30s - warmer: - core: 1 - max: 4 - keep_alive: 5m - search: - #max_queue_size: 1000 #DEPRECATION - queue_size: 1000 - size: 13 - #auto_queue_frame_size: 2000 #DEPRECATION - #target_response_time: 1s #DEPRECATION - #min_queue_size: 1000 #DEPRECATION - fetch_shard_store: - core: 1 - max: 16 - keep_alive: 5m - flush: - core: 1 - max: 4 - keep_alive: 5m - management: - core: 1 - max: 5 - keep_alive: 5m - analyze: - queue_size: 16 - size: 1 - get: - queue_size: 1000 - size: 8 - system_read: - queue_size: 2000 - size: 4 - estimated_time_interval: 200ms - write: - queue_size: 10000 - size: 8 - snapshot: - core: 1 - max: 4 - keep_alive: 5m - search_throttled: - #max_queue_size: 100 #DEPRECATION - queue_size: 100 - size: 1 - #auto_queue_frame_size: 200 #DEPRECATION - #target_response_time: 1s #DEPRECATION - #min_queue_size: 100 #DEPRECATION - index: - codec: default - recovery: - type: "" - store: - type: "" - fs: - fs_lock: native - preload: [] - snapshot: - uncached_chunk_size: "-1b" - cache: - excluded_file_types: [] - monitor: - jvm: - gc: - enabled: true - overhead: - warn: 50 - debug: 10 - info: 25 - refresh_interval: 1s - refresh_interval: 1s - process: - refresh_interval: 1s - os: - refresh_interval: 1s - fs: - health: - enabled: true - refresh_interval: 120s - slow_path_logging_threshold: 5s - refresh_interval: 1s - runtime_fields: - grok: - watchdog: - max_execution_time: 1s - interval: 1s - transport: - tcp: - reuse_address: true - keep_count: "-1" - #connect_timeout: 30s #DEPRECATION - keep_interval: "-1" - #compress: false #DEPRECATION - #port: 9300-9400 #DEPRECATION - no_delay: true - keep_alive: true - receive_buffer_size: "-1b" - keep_idle: "-1" - send_buffer_size: "-1b" - bind_host: - - 0.0.0.0 - connect_timeout: 30s - compress: false - ping_schedule: "-1" - connections_per_node: - recovery: 2 - state: 1 - bulk: 3 - reg: 6 - ping: 1 - tracer: - include: [] - exclude: - - internal:discovery/zen/fd* - - internal:coordination/fault_detection/* - - cluster:monitor/nodes/liveness - type: security4 - slow_operation_logging_threshold: 5s - type.default: netty4 - features: - x-pack: true - port: 9300-9400 - host: [] - publish_port: 9300 - #tcp_no_delay: true #DEPRECATION - publish_host: {{ grains.host }} - netty: - receive_predictor_size: 64kb - receive_predictor_max: 64kb - worker_count: 8 - receive_predictor_min: 64kb - boss_count: 1 + certificate_authorities: + - /usr/share/elasticsearch/config/ca.crt script: - #allowed_contexts: none # ERROR have to set to none - should be list - max_compilations_rate: 20000/1m #DEPRECATION - #cache: #DEPRECATION - #max_size: 100 #DEPRECATION - #expire: 0ms #DEPRECATION - painless: - regex: - enabled: limited - limit-factor: 6 - max_size_in_bytes: 65535 - #allowed_types: none # ERROR have to set to none - should be list - disable_max_compilations_rate: false - indexing_pressure: - memory: - limit: 10% - node: - #data: true #DEPRECATION - # roles: - # - data_frozen - # - data_warm - # - transform ERROR - # - data - # - remote_cluster_client - # - data_cold - # - data_content - # - data_hot - # - ingest - # - master - #max_local_storage_nodes: 1 #DEPRECATION - processors: 8 - store: - allow_mmap: true - #ingest: true #DEPRECATION - #master: true #DEPRECATION - pidfile: "" - #transform: true #DEPRECATION - #remote_cluster_client: true #DEPRECATION - enable_lucene_segment_infos_trace: false - #local_storage: true #DEPRECATION - name: {{ grains.host }} - id: - seed: 0 - #voting_only: false #DEPRECATION - attr: - #transform: ERROR - # node: true ERROR - xpack: - installed: "" - box_type: hot - portsfile: false - #ml: true #DEPRECATION + max_compilations_rate: 20000/1m indices: - replication: - retry_timeout: 60s - initial_retry_backoff_bound: 50ms - cache: - cleanup_interval: 1m - mapping: - dynamic_timeout: 30s - max_in_flight_updates: 10 - memory: - interval: 5s - max_index_buffer_size: "-1" - shard_inactive_time: 5m - index_buffer_size: 10% - min_index_buffer_size: 48mb - breaker: - request: - limit: 60% - type: memory - overhead: 1.0 - total: - limit: 95% - use_real_memory: true - accounting: - limit: 100% - overhead: 1.0 - fielddata: - limit: 40% - type: memory - overhead: 1.03 - type: hierarchy query: bool: - max_nested_depth: 20 max_clause_count: 1500 - query_string: - analyze_wildcard: false - allowLeadingWildcard: true - id_field_data: - enabled: true - recovery: - recovery_activity_timeout: 1800000ms - retry_delay_network: 5s - internal_action_timeout: 15m - retry_delay_state_sync: 500ms - internal_action_long_timeout: 1800000ms - max_concurrent_operations: 1 - max_bytes_per_sec: 40mb - max_concurrent_file_chunks: 2 - requests: - cache: - size: 1% - #expire: 1ms #0ms - ERROR when set to 0ms, set to 1ms and ERROR gone - store: - delete: - shard: - timeout: 30s - analysis: - hunspell: - dictionary: - ignore_case: false - lazy: false - queries: - cache: - count: 10000 - size: 10% - all_segments: false - lifecycle: - history_index_enabled: true - poll_interval: 10m - step: - master_timeout: 30s - fielddata: - cache: - size: "-1b" - plugin: - mandatory: [] - slm: - minimum_interval: 15m - retention_schedule: 0 30 1 * * ? - retention_duration: 1h - history_index_enabled: true - discovery: - #seed_hosts: [] # ERROR - it is forbidden to set both [discovery.seed_hosts] and [discovery.zen.ping.unicast.hosts] - unconfigured_bootstrap_timeout: 3s - request_peers_timeout: 3000ms - zen: - #commit_timeout: 30s #DEPRECATION - #no_master_block: write #DEPRECATION - #join_retry_delay: 100ms #DEPRECATION - #join_retry_attempts: 3 #DEPRECATION - #ping: - #unicast: - #concurrent_connects: 10 # ERROR forbidden to set both [discovery.seed_resolver.max_concurrent_resolvers] and [discovery.zen.ping.unicast.concurrent_connects] - #hosts: [] # ERROR - it is forbidden to set both [discovery.seed_hosts] and [discovery.zen.ping.unicast.hosts] - #hosts.resolve_timeout: 5s # ERROR forbidden to set both [discovery.seed_resolver.timeout] and [discovery.zen.ping.unicast.hosts.resolve_timeout] - #master_election: #DEPRECATION - #ignore_non_master_pings: false #DEPRECATION - #wait_for_joins_timeout: 30000ms #DEPRECATION - #send_leave_request: true #DEPRECATION - ping_timeout: 3s - #bwc_ping_timeout: 3s #DEPRECATION - #join_timeout: 60000ms #DEPRECATION - #publish_diff: - #enable: true #DEPRECATION - #publish: #DEPRECATION - #max_pending_cluster_states: 25 #DEPRECATION - #minimum_master_nodes: "-1" #DEPRECATION - #unsafe_rolling_upgrades_enabled: true #DEPRECATION - #hosts_provider: # ERROR forbidden to set both [discovery.seed_providers] and [discovery.zen.hosts_provider] has to be commented out - #publish_timeout: 30s #DEPRECATION - #fd: #DEPRECATION - #connect_on_network_disconnect: false #DEPRECATION - #ping_interval: 1s #DEPRECATION - #ping_retries: 3 #DEPRECATION - #register_connection_listener: true #DEPRECATION - #ping_timeout: 30s #DEPRECATION - #max_pings_from_another_master: 3 #DEPRECATION - initial_state_timeout: 30s - cluster_formation_warning_timeout: 10000ms - #seed_providers: # ERROR forbidden to set both [discovery.seed_providers] and [discovery.zen.hosts_provider] has to be commented out - type: zen # ERROR java.lang.IllegalArgumentException: node with [discovery.type] set to [single-node] must be master-eligible # test turning off - #seed_resolver: - #max_concurrent_resolvers: 10 # ERROR forbidden to set both [discovery.seed_resolver.max_concurrent_resolvers] and [discovery.zen.ping.unicast.concurrent_connects] - #timeout: 5s # forbidden to set both [discovery.seed_resolver.timeout] and [discovery.zen.ping.unicast.hosts.resolve_timeout] - find_peers_interval: 1000ms - probe: - connect_timeout: 30s - handshake_timeout: 30s - http: - cors: - max-age: 1728000 - allow-origin: "" - allow-headers: X-Requested-With,Content-Type,Content-Length - allow-credentials: false - allow-methods: OPTIONS,HEAD,GET,POST,PUT,DELETE - enabled: false - max_chunk_size: 8kb - compression_level: 3 - max_initial_line_length: 4kb - type: security4 - pipelining: - max_events: 10000 - type.default: netty4 - #content_type: #DEPRECATION - #required: true #DEPRECATION - host: [] - publish_port: "-1" - read_timeout: 0ms - max_content_length: 100mb - netty: - receive_predictor_size: 64kb - max_composite_buffer_components: 69905 - worker_count: 0 - tcp: - reuse_address: true - keep_count: "-1" - keep_interval: "-1" - no_delay: true - keep_alive: true - receive_buffer_size: "-1b" - keep_idle: "-1" - send_buffer_size: "-1b" - bind_host: [] - client_stats: - enabled: true - reset_cookies: false - max_warning_header_count: "-1" - tracer: - include: [] - exclude: [] - max_warning_header_size: "-1b" - detailed_errors: - enabled: true - port: 9200-9300 - max_header_size: 8kb - #tcp_no_delay: true #DEPRECATION - compression: false - publish_host: [] - gateway: - #recover_after_master_nodes: 0 #DEPRECATION - #expected_nodes: "-1" #DEPRECATION - recover_after_data_nodes: "-1" - expected_data_nodes: "-1" - write_dangling_indices_info: true - slow_write_logging_threshold: 10s - recover_after_time: 0ms - #expected_master_nodes: "-1" #DEPRECATION - #recover_after_nodes: "-1" #DEPRECATION - #auto_import_dangling_indices: false #DEPRECATION - snapshot: - refresh_repo_uuid_on_restore: true - max_concurrent_operations: 1000 + + + + From 3ce48acadd1b6e3711e72882c816a0ba723a24bf Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 16 Sep 2021 16:44:31 -0400 Subject: [PATCH 24/31] change cluster_settings to config --- salt/elasticsearch/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 3cb58f71b..f5d983869 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -152,7 +152,7 @@ esyml: - user: 930 - group: 939 - defaults: - ESCONFIG: {{ ESCONFIG.elasticsearch.cluster_settings }} + ESCONFIG: {{ ESCONFIG.elasticsearch.config }} - template: jinja From 0d074dafd4583281d3cf4f2a01fa576f1b723954 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 17 Sep 2021 09:52:50 -0400 Subject: [PATCH 25/31] add missing defaults --- salt/elasticsearch/defaults.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 12d5b27fa..e119fb62d 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -25,7 +25,10 @@ elasticsearch: publish_host: {{ grains.host }} publish_port: 9300 xpack: + ml: + enabled: false security: + enabled: true authc: anonymous: authz_exception: true From d0c87cd31748e2a3b44cd4f27bca07b2ebcc97fb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 17 Sep 2021 12:11:12 -0400 Subject: [PATCH 26/31] allow for pillar override of defaults --- salt/elasticsearch/config.map.jinja | 2 ++ salt/elasticsearch/init.sls | 10 +--------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 81ff4d157..ec05c0e14 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -28,3 +28,5 @@ {% do ESCONFIG.elasticsearch.config.discovery.update({'seed_hosts': [grains.master]}) %} {% endif %} {% endif %} + +{% set ESCONFIG = salt['pillar.get']('elasticsearch:config', default=ESCONFIG.elasticsearch.config, merge=True) %} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index f5d983869..9f475c2c3 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -152,17 +152,9 @@ esyml: - user: 930 - group: 939 - defaults: - ESCONFIG: {{ ESCONFIG.elasticsearch.config }} + ESCONFIG: {{ ESCONFIG }} - template: jinja - -esyml_test: - file.managed: - - name: /tmp/elasticsearch.yml - - user: 930 - - group: 939 - - contents: {{ ESCONFIG | yaml }} - #sync templates to /opt/so/conf/elasticsearch/templates {% for TEMPLATE in TEMPLATES %} es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: From b877aa44bce0fbdb14a03fcfbb26c85d33002199 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 17 Sep 2021 14:10:45 -0400 Subject: [PATCH 27/31] update dict --- salt/elasticsearch/config.map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index ec05c0e14..a6a6ec9c5 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -14,7 +14,7 @@ {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} - {% do ESCONFIG.elasticsearch.config.discovery.update({'seed_hosts': [grains.master]}) %} + {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %} {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} {% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(SN.split('_')|first) %} {% endfor %} @@ -25,7 +25,7 @@ {% else %} {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['data', 'ingest']}) %} {% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': 'hot'}) %} - {% do ESCONFIG.elasticsearch.config.discovery.update({'seed_hosts': [grains.master]}) %} + {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %} {% endif %} {% endif %} From e864bc54044362007b45f88562a06d385eaeb852 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 17 Sep 2021 15:28:35 -0400 Subject: [PATCH 28/31] move custom es cluster name pillar location --- setup/so-functions | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d851c80b1..fbc06fd84 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1725,16 +1725,17 @@ manager_global() { "elastic:"\ " features: False"\ "elasticsearch:"\ - " replicas: 0" >> "$global_pillar" if [ -n "$ESCLUSTERNAME" ]; then printf '%s\n'\ " true_cluster: True"\ - " true_cluster_name: '$ESCLUSTERNAME'" >> "$global_pillar" + " config:"\ + " cluster:"\ + " name: '$ESCLUSTERNAME'" >> "$global_pillar" else printf '%s\n'\ - " true_cluster: False"\ - " true_cluster_name: 'so'" >> "$global_pillar" + " true_cluster: False" >> "$global_pillar" fi + " replicas: 0" >> "$global_pillar" printf '%s\n'\ " discovery_nodes: 1"\ " hot_warm_enabled: False"\ From 109c83d8c3b84cb79e02e312272159828ce5b557 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 17 Sep 2021 15:29:41 -0400 Subject: [PATCH 29/31] move custom es cluster name pillar location --- salt/elasticsearch/config.map.jinja | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index a6a6ec9c5..d0944842c 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -10,8 +10,7 @@ {% endif %} {% if salt['pillar.get']('elasticsearch:true_cluster', False) %} - {% do ESCONFIG.elasticsearch.config.cluster.update({'name': salt['pillar.get']('elasticsearch:true_cluster_name')}) %} {# this is temporary #} - {% if grains.id.split('_') | last in ['manager','managersearch'] %} + {% if grains.id.split('_') | last in ['manager','managersearch'] %} {% if salt['pillar.get']('nodestab', {}) %} {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %} {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %} From 01323cc192721e35316b67eb55e8ab1ea8c9753b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 17 Sep 2021 15:44:54 -0400 Subject: [PATCH 30/31] fix clustername redirect --- setup/so-functions | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index fbc06fd84..06ec7238a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1724,7 +1724,7 @@ manager_global() { " hot_warm: False"\ "elastic:"\ " features: False"\ - "elasticsearch:"\ + "elasticsearch:"\ >> "$global_pillar" if [ -n "$ESCLUSTERNAME" ]; then printf '%s\n'\ " true_cluster: True"\ @@ -1735,8 +1735,9 @@ manager_global() { printf '%s\n'\ " true_cluster: False" >> "$global_pillar" fi - " replicas: 0" >> "$global_pillar" + printf '%s\n'\ + " replicas: 0"\ " discovery_nodes: 1"\ " hot_warm_enabled: False"\ " cluster_routing_allocation_disk.threshold_enabled: true"\ From aed73511e49fc185a0c012f6db24e50e550180d5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 20 Sep 2021 09:24:03 -0400 Subject: [PATCH 31/31] file cleanup, comment cleanup --- salt/elasticsearch/config.map.jinja | 5 -- salt/elasticsearch/files/elasticsearch.yml | 72 ---------------------- 2 files changed, 77 deletions(-) delete mode 100644 salt/elasticsearch/files/elasticsearch.yml diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index d0944842c..1ca729143 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,8 +1,3 @@ -# soup needs -# elasticsearch:esclustername pillar move to elasticsearch:config:cluster:name -# move elasticsearch:true_cluster_name to elasticsearch:config:cluster:name if true_cluster enabled -# elasticsearch:node_route_type moved - {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} {% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml deleted file mode 100644 index 518cd74e9..000000000 --- a/salt/elasticsearch/files/elasticsearch.yml +++ /dev/null @@ -1,72 +0,0 @@ -{%- set NODE_ROUTE_TYPE = salt['pillar.get']('elasticsearch:node_route_type', 'hot') %} -{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip') %} -{%- set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} -{%- if TRUECLUSTER is sameas true %} - {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:true_cluster_name') %} -{%- else %} - {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:esclustername') %} -{%- endif %} -{%- set NODE_ROLES = salt['pillar.get']('elasticsearch:node_roles', ['data', 'ingest']) %} -cluster.name: "{{ ESCLUSTERNAME }}" -network.host: 0.0.0.0 -path.logs: /var/log/elasticsearch -action.destructive_requires_name: true -transport.bind_host: 0.0.0.0 -transport.publish_host: {{ grains.host }} -transport.publish_port: 9300 -cluster.routing.allocation.disk.threshold_enabled: true -cluster.routing.allocation.disk.watermark.low: 95% -cluster.routing.allocation.disk.watermark.high: 98% -cluster.routing.allocation.disk.watermark.flood_stage: 98% -xpack.ml.enabled: false -xpack.security.enabled: true -xpack.security.transport.ssl.enabled: true -xpack.security.transport.ssl.verification_mode: none -xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key -xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt -xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ] -xpack.security.http.ssl.enabled: true -xpack.security.http.ssl.client_authentication: none -xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key -xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt -xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt -{% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} -xpack.security.authc: - anonymous: - username: anonymous_user - roles: superuser - authz_exception: true -{% endif %} -node.name: {{ grains.host }} -script.max_compilations_rate: 20000/1m -{%- if TRUECLUSTER is sameas true %} - {%- if grains.role == 'so-manager' %} - {%- if salt['pillar.get']('nodestab', {}) %} -node.roles: [ master, data, remote_cluster_client ] -discovery.seed_hosts: - - {{ grains.master }} - {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - - {{ SN.split('_')|first }} - {%- endfor %} - {%- endif %} - {%- elif grains.role == 'so-managersearch' %} - {%- if salt['pillar.get']('nodestab', {}) %} -node.roles: [ master, data, remote_cluster_client ] -discovery.seed_hosts: - - {{ grains.master }} - {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - - {{ SN.split('_')|first }} - {%- endfor %} - {%- endif %} -node.attr.box_type: {{ NODE_ROUTE_TYPE }} - {%- else %} -node.roles: {{ NODE_ROLES }} -node.attr.box_type: {{ NODE_ROUTE_TYPE }} -discovery.seed_hosts: - - {{ grains.master }} - {%- endif %} -{%- endif %} -{%- if TRUECLUSTER is sameas false %} -node.attr.box_type: {{ NODE_ROUTE_TYPE }} -{%- endif %} -indices.query.bool.max_clause_count: 1500