diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja new file mode 100644 index 000000000..e6860e699 --- /dev/null +++ b/salt/elasticsearch/config.map.jinja @@ -0,0 +1,5 @@ +{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG %} + +{% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} + {% do ESCONFIG.elasticsearch.defaults.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %} +{% endif %} diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml new file mode 100644 index 000000000..8fc244d6c --- /dev/null +++ b/salt/elasticsearch/defaults.yaml @@ -0,0 +1,1112 @@ +elasticsearch: + es_port: 9200 + esheap: 4049m + esclustername: default-cluster-name + log_size_limit: 95 #used for curator + + index_settings: + so-beats: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-firewall: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-flow: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-ids: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-import: + shards: 1 + warm: 7 + close: 73000 + delete: 73001 + so-osquery: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-ossec: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-strelka: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-syslog: + shards: 1 + warm: 7 + close: 30 + delete: 365 + so-zeek: + shards: 5 + warm: 7 + close: 45 + delete: 365 + + + + persistent: + cluster: + remote: + default-cluster-name: + seeds: + - 127.0.0.1:9300 + transient: + {} + defaults: + cluster: + max_voting_config_exclusions: 10 + auto_shrink_voting_configuration: true + election: + duration: 500ms + initial_timeout: 100ms + max_timeout: 10s + back_off_time: 100ms + strategy: supports_voting_only + no_master_block: write + persistent_tasks: + allocation: + enable: all + recheck_interval: 30s + blocks: + read_only_allow_delete: false + read_only: false + remote: + node: + attr: + initial_connect_timeout: 30s + connect: true + connections_per_cluster: 3 + follower_lag: + timeout: 90000ms + routing: + use_adaptive_replica_selection: true + rebalance: + enable: all + allocation: + node_concurrent_incoming_recoveries: 2 + include: + _tier: + node_initial_primaries_recoveries: 4 + same_shard: + host: false + total_shards_per_node: -1 + require: + _tier: + shard_state: + reroute: + priority: NORMAL + type: balanced + disk: + threshold_enabled: true + watermark: + flood_stage.frozen.max_headroom: 20GB + flood_stage: 98% + high: 98% + low: 95% + enable_for_single_data_node: false + flood_stage.frozen: 95% + include_relocations: true + reroute_interval: 60s + awareness: + attributes: [] + balance: + index: 0.55 + threshold: 1.0 + shard: 0.45 + enable: all + node_concurrent_outgoing_recoveries: 2 + allow_rebalance: indices_all_active + cluster_concurrent_rebalance: 2 + node_concurrent_recoveries: 2 + exclude: + _tier: + indices: + tombstones: + size: 500 + close: + enable: true + max_shards_per_node.frozen: 3000 + nodes: + reconnect_interval: 10s + service: + slow_master_task_logging_threshold: 10s + slow_task_logging_threshold: 30s + publish: + timeout: 30000ms + info_timeout: 10000ms + name: default-cluster-name + fault_detection: + leader_check: + interval: 1000ms + timeout: 10000ms + retry_count: 3 + follower_check: + interval: 1000ms + timeout: 10000ms + retry_count: 3 + join: + timeout: 60000ms + max_shards_per_node: 1000 + initial_master_nodes: [] + snapshot: + info: + max_concurrent_fetches: 5 + info: + update: + interval: 30s + timeout: 15s + stack: + templates: + enabled: true + logger: + level: INFO + bootstrap: + memory_lock: false + system_call_filter: true + ctrlhandler: true + processors: 8 + ingest: + user_agent: + cache_size: 1000 + geoip: + cache_size: 1000 + downloader: + enabled: false + endpoint: https://geoip.elastic.co/v1/database + poll: + interval: 3d + grok: + watchdog: + max_execution_time: 1s + interval: 1s + network: + host: + - 0.0.0.0 + tcp: + reuse_address: true + keep_count: -1 + connect_timeout: 30s + keep_interval: -1 + no_delay: true + keep_alive: true + receive_buffer_size: -1b + keep_idle: -1 + send_buffer_size: -1b + bind_host: + - 0.0.0.0 + server: true + breaker: + inflight_requests: + limit: 100% + overhead: 2.0 + publish_host: + - 0.0.0.0 + pidfile: + path: + data: [] + logs: /var/log/elasticsearch + shared_data: + home: /usr/share/elasticsearch + repo: [] + search: + default_search_timeout: -1 + highlight: + term_vector_multi_value: true + default_allow_partial_results: true + max_open_scroll_context: 500 + max_buckets: 65536 + low_level_cancellation: true + allow_expensive_queries: true + keep_alive_interval: 1m + remote: + node: + attr: + initial_connect_timeout: 30s + connect: true + connections_per_cluster: 3 + default_keep_alive: 5m + max_keep_alive: 24h + aggs: + rewrite_to_filter_by_filter: true + security: + manager: + filter_bad_defaults: true + transform: + task_thread_pool: + queue_size: 4 + size: 4 + ccr: + wait_for_metadata_timeout: 60s + indices: + recovery: + recovery_activity_timeout: 60s + chunk_size: 1mb + internal_action_timeout: 60s + max_bytes_per_sec: 40mb + max_concurrent_file_chunks: 5 + auto_follow: + wait_for_metadata_timeout: 60s + repositories: + fs: + compress: false + chunk_size: 9223372036854775807b + location: + url: + supported_protocols: + - http + - https + - ftp + - file + - jar + allowed_urls: [] + url: http: + action: + auto_create_index: true + search: + shard_count: + limit: 9223372036854775807 + destructive_requires_name: true + client: + type: node + transport: + ignore_cluster_name: false + nodes_sampler_interval: 5s + sniff: false + ping_timeout: 5s + enrich: + max_force_merge_attempts: 3 + cleanup_period: 15m + fetch_size: 10000 + coordinator_proxy: + max_concurrent_requests: 8 + max_lookups_per_request: 128 + queue_capacity: 1024 + max_concurrent_policy_executions: 50 + xpack: + flattened: + enabled: true + watcher: + execution: + scroll: + size: 0 + timeout: + default_throttle_period: 5s + internal: + ops: + bulk: + default_timeout: + index: + default_timeout: + search: + default_timeout: + thread_pool: + queue_size: 1000 + size: 40 + index: + rest: + direct_access: + use_ilm_index_management: true + history: + cleaner_service: + enabled: true + trigger: + schedule: + ticker: + tick_interval: 500ms + enabled: true + input: + search: + default_timeout: + encrypt_sensitive_data: false + transform: + search: + default_timeout: + stop: + timeout: 30s + watch: + scroll: + size: 0 + bulk: + concurrent_requests: 0 + flush_interval: 1s + size: 1mb + actions: 1 + actions: + bulk: + default_timeout: + index: + default_timeout: + eql: + enabled: true + data_frame: + enabled: true + ilm: + enabled: true + monitoring: + migration: + decommission_alerts: false + collection: + cluster: + stats: + timeout: 10s + node: + stats: + timeout: 10s + indices: [] + ccr: + stats: + timeout: 10s + enrich: + stats: + timeout: 10s + index: + stats: + timeout: 10s + recovery: + active_only: false + timeout: 10s + interval: 10s + enabled: false + ml: + job: + stats: + timeout: 10s + history: + duration: 168h + elasticsearch: + collection: + enabled: true + enabled: true + graph: + enabled: true + searchable: + snapshot: + allocate_on_rolling_restart: false + cache: + range_size: 32mb + sync: + max_files: 10000 + interval: 60s + shutdown_timeout: 10s + recovery_range_size: 128kb + shared_cache: + recovery_range_size: 128kb + region_size: 16mb + size: 0 + min_time_delta: 60s + decay: + interval: 60s + size.max_headroom: -1 + range_size: 16mb + max_freq: 100 + rollup: + enabled: true + task_thread_pool: + queue_size: -1 + size: 1 + sql: + enabled: true + searchable_snapshots: + cache_fetch_async_thread_pool: + core: 0 + max: 24 + keep_alive: 30s + cache_prewarming_thread_pool: + core: 0 + max: 16 + keep_alive: 30s + license: + upload: + types: + - standard + - gold + - platinum + - enterprise + - trial + self_generated: + type: basic + logstash: + enabled: true + notification: + pagerduty: + default_account: + email: + default_account: + html: + sanitization: + allow: + - body + - head + - _tables + - _links + - _blocks + - _formatting + - img:embedded + disallow: [] + enabled: true + reporting: + retries: 40 + warning: + enabled: true + interval: 15s + jira: + default_account: + slack: + default_account: + security: + operator_privileges: + enabled: false + dls_fls: + enabled: true + dls: + bitset: + cache: + size: 10% + ttl: 2h + transport: + filter: + allow: [] + deny: [] + enabled: true + ssl: + enabled: true + ssl: + diagnose: + trust: true + enabled: true + crypto: + thread_pool: + queue_size: 1000 + size: 4 + filter: + always_allow_bound_address: true + encryption: + algorithm: AES/CTR/NoPadding + audit: + enabled: false + logfile: + emit_node_id: true + emit_node_host_name: false + emit_node_name: false + events: + emit_request_body: false + include: + - ACCESS_DENIED + - ACCESS_GRANTED + - ANONYMOUS_ACCESS_DENIED + - AUTHENTICATION_FAILED + - CONNECTION_DENIED + - TAMPERED_REQUEST + - RUN_AS_DENIED + - RUN_AS_GRANTED + - SECURITY_CONFIG_CHANGE + exclude: + [] + emit_node_host_address: false + authc: + password_hashing: + algorithm: bcrypt + success_cache: + size: 10000 + enabled: true + expire_after_access: 1h + api_key: + doc_cache: + ttl: 5m + cache: + hash_algo: ssha256 + max_keys: 10000 + ttl: 24h + delete: + interval: 24h + timeout: -1 + enabled: false + hashing: + algorithm: pbkdf2 + anonymous: + authz_exception: true + roles: + - superuser + username: anonymous_user + run_as: + enabled: true + reserved_realm: + enabled: true + service_token: + cache: + hash_algo: ssha256 + max_tokens: 100000 + ttl: 20m + token: + delete: + interval: 30m + timeout: -1 + enabled: false + thread_pool: + queue_size: 1000 + size: 1 + timeout: 20m + fips_mode: + enabled: false + encryption_key: + length: 128 + algorithm: AES + http: + filter: + allow: [] + deny: [] + enabled: true + ssl: + enabled: true + automata: + max_determinized_states: 100000 + cache: + size: 10000 + ttl: 48h + enabled: true + user: null + authz: + store: + privileges: + cache: + ttl: 24h + max_size: 10000 + roles: + index: + cache: + ttl: 20m + max_size: 10000 + cache: + max_size: 10000 + negative_lookup_cache: + max_size: 10000 + field_permissions: + cache: + max_size_in_bytes: 104857600 + transform: + num_transform_failure_retries: 10 + enabled: true + vectors: + enabled: true + ccr: + enabled: true + ccr_thread_pool: + queue_size: 100 + size: 32 + idp: + privileges: + application: + cache: + size: 100 + ttl: 90m + metadata: + signing: + keystore: + alias: + slo_endpoint: + post: https: + redirect: https: + defaults: + nameid_format: urn:oasis:names:tc:SAML:2.0:nameid-format:transient + authn_expiry: 5m + allowed_nameid_formats: + - urn:oasis:names:tc:SAML:2.0:nameid-format:transient + contact: + given_name: + email: + surname: + organization: + display_name: + name: + url: http: + sso_endpoint: + post: https: + redirect: https: + entity_id: + signing: + keystore: + alias: + sp: + cache: + size: 1000 + ttl: 60m + wildcard: + path: wildcard_services.json + enabled: false + slm: + enabled: true + enrich: + enabled: true + http: + default_connection_timeout: 10s + proxy: + host: + scheme: + port: 0 + whitelist: + - * + default_read_timeout: 10s + max_response_size: 10mb + autoscaling: + memory: + monitor: + timeout: 15s + ml: + max_anomaly_records: 500 + enable_config_migration: true + max_open_jobs: 512 + min_disk_space_off_heap: 5gb + use_auto_machine_memory_percent: false + inference_model: + cache_size: 40% + time_to_live: 5m + nightly_maintenance_requests_per_second: -1.0 + node_concurrent_job_allocations: 2 + max_model_memory_limit: 0b + enabled: false + max_lazy_ml_nodes: 0 + max_ml_node_size: 0b + max_machine_memory_percent: 30 + persist_results_max_retries: 20 + autodetect_process: true + max_inference_processors: 50 + process_connect_timeout: 10s + rest: + action: + multi: + allow_explicit_index: true + cache: + recycler: + page: + limit: + heap: 10% + type: CONCURRENT + weight: + longs: 1.0 + ints: 1.0 + bytes: 1.0 + objects: 0.1 + async_search: + index_cleanup_interval: 1h + reindex: + remote: + whitelist: [] + resource: + reload: + enabled: true + interval: + low: 60s + high: 5s + medium: 30s + thread_pool: + force_merge: + queue_size: -1 + size: 1 + fetch_shard_started: + core: 1 + max: 16 + keep_alive: 5m + listener: + queue_size: -1 + size: 4 + refresh: + core: 1 + max: 4 + keep_alive: 5m + system_write: + queue_size: 1000 + size: 4 + generic: + core: 4 + max: 128 + keep_alive: 30s + warmer: + core: 1 + max: 4 + keep_alive: 5m + search: + max_queue_size: 1000 + queue_size: 1000 + size: 13 + auto_queue_frame_size: 2000 + target_response_time: 1s + min_queue_size: 1000 + fetch_shard_store: + core: 1 + max: 16 + keep_alive: 5m + flush: + core: 1 + max: 4 + keep_alive: 5m + management: + core: 1 + max: 5 + keep_alive: 5m + analyze: + queue_size: 16 + size: 1 + get: + queue_size: 1000 + size: 8 + system_read: + queue_size: 2000 + size: 4 + estimated_time_interval: 200ms + write: + queue_size: 10000 + size: 8 + snapshot: + core: 1 + max: 4 + keep_alive: 5m + search_throttled: + max_queue_size: 100 + queue_size: 100 + size: 1 + auto_queue_frame_size: 200 + target_response_time: 1s + min_queue_size: 100 + index: + codec: default + recovery: + type: + store: + type: + fs: + fs_lock: native + preload: [] + snapshot: + uncached_chunk_size: -1b + cache: + excluded_file_types: [] + monitor: + jvm: + gc: + enabled: true + overhead: + warn: 50 + debug: 10 + info: 25 + refresh_interval: 1s + refresh_interval: 1s + process: + refresh_interval: 1s + os: + refresh_interval: 1s + fs: + health: + enabled: true + refresh_interval: 120s + slow_path_logging_threshold: 5s + refresh_interval: 1s + runtime_fields: + grok: + watchdog: + max_execution_time: 1s + interval: 1s + transport: + tcp: + reuse_address: true + keep_count: -1 + connect_timeout: 30s + keep_interval: -1 + compress: false + port: 9300-9400 + no_delay: true + keep_alive: true + receive_buffer_size: -1b + keep_idle: -1 + send_buffer_size: -1b + bind_host: + - 0.0.0.0 + connect_timeout: 30s + compress: false + ping_schedule: -1 + connections_per_node: + recovery: 2 + state: 1 + bulk: 3 + reg: 6 + ping: 1 + tracer: + include: [] + exclude: + - internal:discovery/zen/fd* + - internal:coordination/fault_detection/* + - cluster:monitor/nodes/liveness + type: security4 + slow_operation_logging_threshold: 5s + type.default: netty4 + features: + x-pack: true + port: 9300-9400 + host: [] + publish_port: 9300 + tcp_no_delay: true + publish_host: {{ grains.host }} + netty: + receive_predictor_size: 64kb + receive_predictor_max: 64kb + worker_count: 8 + receive_predictor_min: 64kb + boss_count: 1 + script: + allowed_contexts: [] + max_compilations_rate: 20000/1m + cache: + max_size: 100 + expire: 0ms + painless: + regex: + enabled: limited + limit-factor: 6 + max_size_in_bytes: 65535 + allowed_types: [] + disable_max_compilations_rate: false + indexing_pressure: + memory: + limit: 10% + node: + data: true + roles: + - data_frozen + - data_warm + - transform + - data + - remote_cluster_client + - data_cold + - data_content + - data_hot + - ingest + - master + max_local_storage_nodes: 1 + processors: 8 + store: + allow_mmap: true + ingest: true + master: true + pidfile: + transform: true + remote_cluster_client: true + enable_lucene_segment_infos_trace: false + local_storage: true + name: {{ grains.host }} + id: + seed: 0 + voting_only: false + attr: + transform: + node: true + xpack: + installed: true + box_type: hot + portsfile: false + ml: true + indices: + replication: + retry_timeout: 60s + initial_retry_backoff_bound: 50ms + cache: + cleanup_interval: 1m + mapping: + dynamic_timeout: 30s + max_in_flight_updates: 10 + memory: + interval: 5s + max_index_buffer_size: -1 + shard_inactive_time: 5m + index_buffer_size: 10% + min_index_buffer_size: 48mb + breaker: + request: + limit: 60% + type: memory + overhead: 1.0 + total: + limit: 95% + use_real_memory: true + accounting: + limit: 100% + overhead: 1.0 + fielddata: + limit: 40% + type: memory + overhead: 1.03 + type: hierarchy + query: + bool: + max_nested_depth: 20 + max_clause_count: 1500 + query_string: + analyze_wildcard: false + allowLeadingWildcard: true + id_field_data: + enabled: true + recovery: + recovery_activity_timeout: 1800000ms + retry_delay_network: 5s + internal_action_timeout: 15m + retry_delay_state_sync: 500ms + internal_action_long_timeout: 1800000ms + max_concurrent_operations: 1 + max_bytes_per_sec: 40mb + max_concurrent_file_chunks: 2 + requests: + cache: + size: 1% + expire: 0ms + store: + delete: + shard: + timeout: 30s + analysis: + hunspell: + dictionary: + ignore_case: false + lazy: false + queries: + cache: + count: 10000 + size: 10% + all_segments: false + lifecycle: + history_index_enabled: true + poll_interval: 10m + step: + master_timeout: 30s + fielddata: + cache: + size: -1b + plugin: + mandatory: [] + slm: + minimum_interval: 15m + retention_schedule: 0 30 1 * * ? + retention_duration: 1h + history_index_enabled: true + discovery: + seed_hosts: [] + unconfigured_bootstrap_timeout: 3s + request_peers_timeout: 3000ms + zen: + commit_timeout: 30s + no_master_block: write + join_retry_delay: 100ms + join_retry_attempts: 3 + ping: + unicast: + concurrent_connects: 10 + hosts: [] + hosts.resolve_timeout: 5s + master_election: + ignore_non_master_pings: false + wait_for_joins_timeout: 30000ms + send_leave_request: true + ping_timeout: 3s + bwc_ping_timeout: 3s + join_timeout: 60000ms + publish_diff: + enable: true + publish: + max_pending_cluster_states: 25 + minimum_master_nodes: -1 + unsafe_rolling_upgrades_enabled: true + hosts_provider: [] + publish_timeout: 30s + fd: + connect_on_network_disconnect: false + ping_interval: 1s + ping_retries: 3 + register_connection_listener: true + ping_timeout: 30s + max_pings_from_another_master: 3 + initial_state_timeout: 30s + cluster_formation_warning_timeout: 10000ms + seed_providers: [] + type: single-node + seed_resolver: + max_concurrent_resolvers: 10 + timeout: 5s + find_peers_interval: 1000ms + probe: + connect_timeout: 30s + handshake_timeout: 30s + http: + cors: + max-age: 1728000 + allow-origin: + allow-headers: X-Requested-With,Content-Type,Content-Length + allow-credentials: false + allow-methods: OPTIONS,HEAD,GET,POST,PUT,DELETE + enabled: false + max_chunk_size: 8kb + compression_level: 3 + max_initial_line_length: 4kb + type: security4 + pipelining: + max_events: 10000 + type.default: netty4 + content_type: + required: true + host: [] + publish_port: -1 + read_timeout: 0ms + max_content_length: 100mb + netty: + receive_predictor_size: 64kb + max_composite_buffer_components: 69905 + worker_count: 0 + tcp: + reuse_address: true + keep_count: -1 + keep_interval: -1 + no_delay: true + keep_alive: true + receive_buffer_size: -1b + keep_idle: -1 + send_buffer_size: -1b + bind_host: [] + client_stats: + enabled: true + reset_cookies: false + max_warning_header_count: -1 + tracer: + include: [] + exclude: [] + max_warning_header_size: -1b + detailed_errors: + enabled: true + port: 9200-9300 + max_header_size: 8kb + tcp_no_delay: true + compression: false + publish_host: [] + gateway: + recover_after_master_nodes: 0 + expected_nodes: -1 + recover_after_data_nodes: -1 + expected_data_nodes: -1 + write_dangling_indices_info: true + slow_write_logging_threshold: 10s + recover_after_time: 0ms + expected_master_nodes: -1 + recover_after_nodes: -1 + auto_import_dangling_indices: false + snapshot: + refresh_repo_uuid_on_restore: true + max_concurrent_operations: 1000 diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 4045fa10f..09cba56f9 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -15,6 +15,8 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} + + {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} @@ -36,6 +38,7 @@ {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} +{% from 'elasticsearch/config.map.jinja' import ESCONFIG with context %} vm.max_map_count: @@ -142,6 +145,14 @@ esyml: - group: 939 - template: jinja +esyml_test: + file.managed: + - name: /tmp/elasticsearch.yml + - source: salt://elasticsearch/files/elasticsearch.yml + - user: 930 + - group: 939 + - contents: {{ ESCONFIG | yaml }} + #sync templates to /opt/so/conf/elasticsearch/templates {% for TEMPLATE in TEMPLATES %} es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: