diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index e7a9a0491..206c2fad6 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -46,6 +46,7 @@ 'pcap', 'suricata', 'healthcheck', + 'elasticagent', 'schedule', 'tcpreplay', 'docker_clean' diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 21b94a2ba..2e5e28a6c 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -178,6 +178,11 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] + 'so-elastic-agent': + final_octet: 46 + custom_bind_mounts: [] + extra_hosts: [] + extra_env: [] 'so-telegraf': final_octet: 99 custom_bind_mounts: [] diff --git a/salt/elasticagent/config.sls b/salt/elasticagent/config.sls new file mode 100644 index 000000000..18d0e482e --- /dev/null +++ b/salt/elasticagent/config.sls @@ -0,0 +1,47 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% if sls.split('.')[0] in allowed_states %} + +# Add EA Group +elasticagentgroup: + group.present: + - name: elastic-agent + - gid: 949 + +# Add EA user +elastic-agent: + user.present: + - uid: 949 + - gid: 949 + - home: /opt/so/conf/elastic-agent + - createhome: False + +elasticagentconfdir: + file.directory: + - name: /opt/so/conf/elastic-agent + - user: 949 + - group: 939 + - makedirs: True + +# Create config +create-elastic-agent-config: + file.managed: + - name: /opt/so/conf/elastic-agent/elastic-agent.yml + - source: salt://elasticagent/files/elastic-agent.yml.jinja + - user: 949 + - group: 939 + - template: jinja + + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticagent/defaults.yaml b/salt/elasticagent/defaults.yaml new file mode 100644 index 000000000..f9b1bd67a --- /dev/null +++ b/salt/elasticagent/defaults.yaml @@ -0,0 +1,2 @@ +elasticagent: + enabled: False \ No newline at end of file diff --git a/salt/elasticagent/disabled.sls b/salt/elasticagent/disabled.sls new file mode 100644 index 000000000..a6d72b193 --- /dev/null +++ b/salt/elasticagent/disabled.sls @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} + +include: + - elasticagent.sostatus + +so-elastic-agent: + docker_container.absent: + - force: True + +so-elastic-agent_so-status.disabled: + file.comment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-elastic-agent$ + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticagent/enabled.sls b/salt/elasticagent/enabled.sls new file mode 100644 index 000000000..ee4b1d2be --- /dev/null +++ b/salt/elasticagent/enabled.sls @@ -0,0 +1,62 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'docker/docker.map.jinja' import DOCKER %} + + +include: + - elasticagent.config + - elasticagent.sostatus + +so-elastic-agent: + docker_container.running: + - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }} + - name: so-elastic-agent + - hostname: {{ GLOBALS.hostname }} + - detach: True + - user: 949 + - networks: + - sobridge: + - ipv4_address: {{ DOCKER.containers['so-elastic-agent'].ip }} + - extra_hosts: + - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} + - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }} + {% if DOCKER.containers['so-elastic-agent'].extra_hosts %} + {% for XTRAHOST in DOCKER.containers['so-elastic-agent'].extra_hosts %} + - {{ XTRAHOST }} + {% endfor %} + {% endif %} + - binds: + - /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro + - /nsm:/nsm:ro + {% if DOCKER.containers['so-elastic-agent'].custom_bind_mounts %} + {% for BIND in DOCKER.containers['so-elastic-agent'].custom_bind_mounts %} + - {{ BIND }} + {% endfor %} + {% endif %} + {% if DOCKER.containers['so-elastic-agent'].extra_env %} + - environment: + {% for XTRAENV in DOCKER.containers['so-elastic-agent'].extra_env %} + - {{ XTRAENV }} + {% endfor %} + {% endif %} + + +delete_so-elastic-agent_so-status.disabled: + file.uncomment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-elastic-agent$ + + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticagent/files/elastic-agent.yml.jinja b/salt/elasticagent/files/elastic-agent.yml.jinja new file mode 100644 index 000000000..45bac49b4 --- /dev/null +++ b/salt/elasticagent/files/elastic-agent.yml.jinja @@ -0,0 +1,119 @@ +{% from 'vars/globals.map.jinja' import GLOBALS %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} + +id: aea1ba80-1065-11ee-a369-97538913b6a9 +revision: 2 +outputs: + default: + type: elasticsearch + hosts: + - 'https://{{ GLOBALS.hostname }}:9200' + username: '{{ ES_USER }}' + password: '{{ ES_PASS }}' + ssl.verification_mode: none +output_permissions: {} +agent: + download: + sourceURI: 'http://{{ GLOBALS.manager }}:8443/artifacts/' + monitoring: + enabled: false + logs: false + metrics: false + features: {} +inputs: + - id: logfile-logs-80ffa884-2cfc-459a-964a-34df25714d85 + name: suricata-logs + revision: 1 + type: logfile + use_output: default + meta: + package: + name: log + version: + data_stream: + namespace: so + package_policy_id: 80ffa884-2cfc-459a-964a-34df25714d85 + streams: + - id: logfile-log.log-80ffa884-2cfc-459a-964a-34df25714d85 + data_stream: + dataset: suricata + paths: + - /nsm/suricata/eve*.json + processors: + - add_fields: + target: event + fields: + category: network + module: suricata + pipeline: suricata.common + - id: logfile-logs-90103ac4-f6bd-4a4a-b596-952c332390fc + name: strelka-logs + revision: 1 + type: logfile + use_output: default + meta: + package: + name: log + version: + data_stream: + namespace: so + package_policy_id: 90103ac4-f6bd-4a4a-b596-952c332390fc + streams: + - id: logfile-log.log-90103ac4-f6bd-4a4a-b596-952c332390fc + data_stream: + dataset: strelka + paths: + - /nsm/strelka/log/strelka.log + processors: + - add_fields: + target: event + fields: + category: file + module: strelka + pipeline: strelka.file + - id: logfile-logs-6197fe84-9b58-4d9b-8464-3d517f28808d + name: zeek-logs + revision: 1 + type: logfile + use_output: default + meta: + package: + name: log + version: + data_stream: + namespace: so + package_policy_id: 6197fe84-9b58-4d9b-8464-3d517f28808d + streams: + - id: logfile-log.log-6197fe84-9b58-4d9b-8464-3d517f28808d + data_stream: + dataset: zeek + paths: + - /nsm/zeek/logs/current/*.log + processors: + - dissect: + tokenizer: '/nsm/zeek/logs/current/%{pipeline}.log' + field: log.file.path + trim_chars: .log + target_prefix: '' + - script: + lang: javascript + source: | + function process(event) { + var pl = event.Get("pipeline"); + event.Put("@metadata.pipeline", "zeek." + pl); + } + - add_fields: + target: event + fields: + category: network + module: zeek + - add_tags: + tags: ics + when: + regexp: + pipeline: >- + ^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm* + exclude_files: + - >- + broker|capture_loss|cluster|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout.log$ diff --git a/salt/elasticagent/init.sls b/salt/elasticagent/init.sls new file mode 100644 index 000000000..3ea474026 --- /dev/null +++ b/salt/elasticagent/init.sls @@ -0,0 +1,13 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'elasticagent/map.jinja' import ELASTICAGENTMERGED %} + +include: +{% if ELASTICAGENTMERGED.enabled %} + - elasticagent.enabled +{% else %} + - elasticagent.disabled +{% endif %} diff --git a/salt/elasticagent/map.jinja b/salt/elasticagent/map.jinja new file mode 100644 index 000000000..f48172502 --- /dev/null +++ b/salt/elasticagent/map.jinja @@ -0,0 +1,7 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + +{% import_yaml 'elasticagent/defaults.yaml' as ELASTICAGENTDEFAULTS %} +{% set ELASTICAGENTMERGED = salt['pillar.get']('elasticagent', ELASTICAGENTDEFAULTS.elasticagent, merge=True) %} diff --git a/salt/elasticagent/sostatus.sls b/salt/elasticagent/sostatus.sls new file mode 100644 index 000000000..e2cbc9b8e --- /dev/null +++ b/salt/elasticagent/sostatus.sls @@ -0,0 +1,21 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} + +append_so-elastic-agent_so-status.conf: + file.append: + - name: /opt/so/conf/so-status/so-status.conf + - text: so-elastic-agent + - unless: grep -q so-elastic-agent$ /opt/so/conf/so-status/so-status.conf + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticagent/tools/sbin/so-elastic-agent-restart b/salt/elasticagent/tools/sbin/so-elastic-agent-restart new file mode 100755 index 000000000..cb125ce10 --- /dev/null +++ b/salt/elasticagent/tools/sbin/so-elastic-agent-restart @@ -0,0 +1,10 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +/usr/sbin/so-restart elastic-agent $1 diff --git a/salt/elasticagent/tools/sbin/so-elastic-agent-start b/salt/elasticagent/tools/sbin/so-elastic-agent-start new file mode 100755 index 000000000..294b9ba7a --- /dev/null +++ b/salt/elasticagent/tools/sbin/so-elastic-agent-start @@ -0,0 +1,12 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-common + +/usr/sbin/so-start elastic-agent $1 diff --git a/salt/elasticagent/tools/sbin/so-elastic-agent-stop b/salt/elasticagent/tools/sbin/so-elastic-agent-stop new file mode 100755 index 000000000..cbfdcd380 --- /dev/null +++ b/salt/elasticagent/tools/sbin/so-elastic-agent-stop @@ -0,0 +1,12 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-common + +/usr/sbin/so-stop elastic-agent $1 diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index cc3b0675f..9bfb6e34d 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -8,13 +8,13 @@ {% if sls.split('.')[0] in allowed_states %} # Add EA Group -elasticsagentgroup: +elasticfleetgroup: group.present: - - name: elastic-agent + - name: elastic-fleet - gid: 947 # Add EA user -elastic-agent: +elastic-fleet: user.present: - uid: 947 - gid: 947 diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json new file mode 100644 index 000000000..4c22f92ee --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json @@ -0,0 +1,106 @@ +{ + "package": { + "name": "elasticsearch", + "version": "" + }, + "name": "elasticsearch-logs", + "namespace": "default", + "description": "Elasticsearch Logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "elasticsearch-logfile": { + "enabled": true, + "streams": { + "elasticsearch.audit": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/*_audit.json" + ] + } + }, + "elasticsearch.deprecation": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/*_deprecation.json" + ] + } + }, + "elasticsearch.gc": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/gc.log.[0-9]*", + "/var/log/elasticsearch/gc.log" + ] + } + }, + "elasticsearch.server": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/elasticsearch/*.log" + ] + } + }, + "elasticsearch.slowlog": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/*_index_search_slowlog.json", + "/var/log/elasticsearch/*_index_indexing_slowlog.json" + ] + } + } + } + }, + "elasticsearch-elasticsearch/metrics": { + "enabled": false, + "vars": { + "hosts": [ + "http://localhost:9200" + ], + "scope": "node" + }, + "streams": { + "elasticsearch.stack_monitoring.ccr": { + "enabled": false + }, + "elasticsearch.stack_monitoring.cluster_stats": { + "enabled": false + }, + "elasticsearch.stack_monitoring.enrich": { + "enabled": false + }, + "elasticsearch.stack_monitoring.index": { + "enabled": false + }, + "elasticsearch.stack_monitoring.index_recovery": { + "enabled": false, + "vars": { + "active.only": true + } + }, + "elasticsearch.stack_monitoring.index_summary": { + "enabled": false + }, + "elasticsearch.stack_monitoring.ml_job": { + "enabled": false + }, + "elasticsearch.stack_monitoring.node": { + "enabled": false + }, + "elasticsearch.stack_monitoring.node_stats": { + "enabled": false + }, + "elasticsearch.stack_monitoring.pending_tasks": { + "enabled": false + }, + "elasticsearch.stack_monitoring.shard": { + "enabled": false + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/idh-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json similarity index 97% rename from salt/elasticfleet/files/integrations/grid-nodes/idh-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json index 95b72e0a0..32055112a 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/idh-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json @@ -6,7 +6,7 @@ "name": "idh-logs", "namespace": "so", "description": "IDH integration", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json similarity index 98% rename from salt/elasticfleet/files/integrations/grid-nodes/import-evtx-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 5bebfd54d..d9f8daeb9 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -6,7 +6,7 @@ "name": "import-evtx-logs", "namespace": "so", "description": "Import Windows EVTX logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "vars": {}, "inputs": { "logs-logfile": { diff --git a/salt/elasticfleet/files/integrations/grid-nodes/import-suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json similarity index 95% rename from salt/elasticfleet/files/integrations/grid-nodes/import-suricata-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json index 4cba27121..f17ee33d1 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/import-suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json @@ -6,7 +6,7 @@ "name": "import-suricata-logs", "namespace": "so", "description": "Import Suricata logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json new file mode 100644 index 000000000..c342b57bd --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "" + }, + "name": "kratos-logs", + "namespace": "so", + "description": "Kratos logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/kratos/kratos.log" + ], + "data_stream.dataset": "kratos", + "tags": ["so-kratos"], + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos", + "custom": "pipeline: kratos" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json new file mode 100644 index 000000000..0349c9fc3 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json @@ -0,0 +1,20 @@ +{ + "package": { + "name": "osquery_manager", + "version": "" + }, + "name": "osquery-grid-nodes", + "namespace": "default", + "policy_id": "so-grid-nodes_general", + "inputs": { + "osquery_manager-osquery": { + "enabled": true, + "streams": { + "osquery_manager.result": { + "enabled": true, + "vars": {} + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json new file mode 100644 index 000000000..6b9cbffaf --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json @@ -0,0 +1,76 @@ +{ + "package": { + "name": "redis", + "version": "" + }, + "name": "redis-logs", + "namespace": "default", + "description": "Redis logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "redis-logfile": { + "enabled": true, + "streams": { + "redis.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/redis/redis.log" + ], + "tags": [ + "redis-log" + ], + "preserve_original_event": false + } + } + } + }, + "redis-redis": { + "enabled": false, + "streams": { + "redis.slowlog": { + "enabled": false, + "vars": { + "hosts": [ + "127.0.0.1:6379" + ], + "password": "" + } + } + } + }, + "redis-redis/metrics": { + "enabled": false, + "vars": { + "hosts": [ + "127.0.0.1:6379" + ], + "idle_timeout": "20s", + "maxconn": 10, + "network": "tcp", + "password": "" + }, + "streams": { + "redis.info": { + "enabled": false, + "vars": { + "period": "10s" + } + }, + "redis.key": { + "enabled": false, + "vars": { + "key.patterns": "- limit: 20\n pattern: *\n", + "period": "10s" + } + }, + "redis.keyspace": { + "enabled": false, + "vars": { + "period": "10s" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json new file mode 100644 index 000000000..84e9ae94d --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "" + }, + "name": "soc-auth-sync-logs", + "namespace": "so", + "description": "Security Onion - Elastic Auth Sync - Logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/soc/sync.log" + ], + "data_stream.dataset": "soc", + "tags": ["so-soc"], + "processors": "- dissect:\n tokenizer: \"%{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: auth_sync", + "custom": "pipeline: common" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json new file mode 100644 index 000000000..07bd89b89 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "" + }, + "name": "soc-salt-relay-logs", + "namespace": "so", + "description": "Security Onion - Salt Relay - Logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/soc/salt-relay.log" + ], + "data_stream.dataset": "soc", + "tags": ["so-soc"], + "processors": "- dissect:\n tokenizer: \"%{soc.ts} | %{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: salt_relay", + "custom": "pipeline: common" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json new file mode 100644 index 000000000..bee14ebf5 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "" + }, + "name": "soc-sensoroni-logs", + "namespace": "so", + "description": "Security Onion - Sensoroni - Logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/sensoroni/sensoroni.log" + ], + "data_stream.dataset": "soc", + "tags": [], + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"sensoroni\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: sensoroni\n- rename:\n fields:\n - from: \"sensoroni.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"sensoroni.fields.status\"\n to: \"http.response.status_code\"\n - from: \"sensoroni.fields.method\"\n to: \"http.request.method\"\n - from: \"sensoroni.fields.path\"\n to: \"url.path\"\n - from: \"sensoroni.message\"\n to: \"event.action\"\n - from: \"sensoroni.level\"\n to: \"log.level\"\n ignore_missing: true", + "custom": "pipeline: common" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json new file mode 100644 index 000000000..285d79148 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "" + }, + "name": "soc-server-logs", + "namespace": "so", + "description": "Security Onion Console Logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/soc/sensoroni-server.log" + ], + "data_stream.dataset": "soc", + "tags": ["so-soc"], + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"soc\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: server\n- rename:\n fields:\n - from: \"soc.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"soc.fields.status\"\n to: \"http.response.status_code\"\n - from: \"soc.fields.method\"\n to: \"http.request.method\"\n - from: \"soc.fields.path\"\n to: \"url.path\"\n - from: \"soc.message\"\n to: \"event.action\"\n - from: \"soc.level\"\n to: \"log.level\"\n ignore_missing: true", + "custom": "pipeline: common" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/strelka-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json similarity index 94% rename from salt/elasticfleet/files/integrations/grid-nodes/strelka-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json index ac6157638..6f6beca99 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/strelka-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json @@ -6,7 +6,7 @@ "name": "strelka-logs", "namespace": "so", "description": "Strelka logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json similarity index 94% rename from salt/elasticfleet/files/integrations/grid-nodes/suricata-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json index 9d7e4040d..7ff43c3a8 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json @@ -6,7 +6,7 @@ "name": "suricata-logs", "namespace": "so", "description": "Suricata integration", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/syslog-tcp-514.json b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json similarity index 94% rename from salt/elasticfleet/files/integrations/grid-nodes/syslog-tcp-514.json rename to salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json index 495aaa309..80baa45ca 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/syslog-tcp-514.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json @@ -6,7 +6,7 @@ "name": "syslog-tcp-514", "namespace": "so", "description": "Syslog Over TCP Port 514", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "inputs": { "tcp-tcp": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/syslog-udp-514.json b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json similarity index 95% rename from salt/elasticfleet/files/integrations/grid-nodes/syslog-udp-514.json rename to salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json index 053e95299..653c788b5 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/syslog-udp-514.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json @@ -6,7 +6,7 @@ "name": "syslog-udp-514", "namespace": "so", "description": "Syslog over UDP Port 514", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_general", "inputs": { "udp-udp": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json new file mode 100644 index 000000000..a5c4c3e81 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json @@ -0,0 +1,40 @@ +{ + "policy_id": "so-grid-nodes_general", + "package": { + "name": "system", + "version": "" + }, + "name": "system-grid-nodes", + "namespace": "default", + "inputs": { + "system-logfile": { + "enabled": true, + "streams": { + "system.auth": { + "enabled": true, + "vars": { + "paths": [ + "/var/log/auth.log*", + "/var/log/secure*" + ] + } + }, + "system.syslog": { + "enabled": true, + "vars": { + "paths": [ + "/var/log/messages*", + "/var/log/syslog*" + ] + } + } + } + }, + "system-winlog": { + "enabled": false + }, + "system-system/metrics": { + "enabled": false + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/elasticsearch-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-logs.json similarity index 98% rename from salt/elasticfleet/files/integrations/grid-nodes/elasticsearch-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-logs.json index 99f2733c9..711602775 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/elasticsearch-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-logs.json @@ -6,7 +6,7 @@ "name": "elasticsearch-logs", "namespace": "default", "description": "Elasticsearch Logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "elasticsearch-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json similarity index 95% rename from salt/elasticfleet/files/integrations/grid-nodes/kratos-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json index 5e134f1f6..c9e4183de 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/kratos-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json @@ -6,7 +6,7 @@ "name": "kratos-logs", "namespace": "so", "description": "Kratos logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/osquery-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json similarity index 89% rename from salt/elasticfleet/files/integrations/grid-nodes/osquery-grid-nodes.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json index 197526ce3..d0281c111 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/osquery-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json @@ -5,7 +5,7 @@ }, "name": "osquery-grid-nodes", "namespace": "default", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "osquery_manager-osquery": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/redis-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/redis-logs.json similarity index 97% rename from salt/elasticfleet/files/integrations/grid-nodes/redis-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/redis-logs.json index a5d4102df..cddcedfd8 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/redis-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/redis-logs.json @@ -6,7 +6,7 @@ "name": "redis-logs", "namespace": "default", "description": "Redis logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "redis-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json similarity index 95% rename from salt/elasticfleet/files/integrations/grid-nodes/soc-auth-sync-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json index 7f60d1706..2004c8c5d 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/soc-auth-sync-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json @@ -6,7 +6,7 @@ "name": "soc-auth-sync-logs", "namespace": "so", "description": "Security Onion - Elastic Auth Sync - Logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json similarity index 95% rename from salt/elasticfleet/files/integrations/grid-nodes/soc-salt-relay-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json index 7821f4081..b1b6098c1 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/soc-salt-relay-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json @@ -6,7 +6,7 @@ "name": "soc-salt-relay-logs", "namespace": "so", "description": "Security Onion - Salt Relay - Logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json similarity index 97% rename from salt/elasticfleet/files/integrations/grid-nodes/soc-sensoroni-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json index 56069ed65..5954e5052 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/soc-sensoroni-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json @@ -6,7 +6,7 @@ "name": "soc-sensoroni-logs", "namespace": "so", "description": "Security Onion - Sensoroni - Logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json similarity index 97% rename from salt/elasticfleet/files/integrations/grid-nodes/soc-server-logs.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json index fcdfc9344..89e26563a 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/soc-server-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json @@ -6,7 +6,7 @@ "name": "soc-server-logs", "namespace": "so", "description": "Security Onion Console Logs", - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "inputs": { "logs-logfile": { "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json similarity index 95% rename from salt/elasticfleet/files/integrations/grid-nodes/system-grid-nodes.json rename to salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json index 3c10227ca..31d30d4e0 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json @@ -1,5 +1,5 @@ { - "policy_id": "so-grid-nodes", + "policy_id": "so-grid-nodes_heavy", "package": { "name": "system", "version": "" diff --git a/salt/elasticfleet/install_agent_grid.sls b/salt/elasticfleet/install_agent_grid.sls index 00a74437e..1d6c5a241 100644 --- a/salt/elasticfleet/install_agent_grid.sls +++ b/salt/elasticfleet/install_agent_grid.sls @@ -2,15 +2,24 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. -{%- set GRIDNODETOKEN = salt['pillar.get']('global:fleet_grid_enrollment_token') -%} +{%- set GRIDNODETOKENGENERAL = salt['pillar.get']('global:fleet_grid_enrollment_token_general') -%} +{%- set GRIDNODETOKENHEAVY = salt['pillar.get']('global:fleet_grid_enrollment_token_heavy') -%} {% set AGENT_STATUS = salt['service.available']('elastic-agent') %} {% if not AGENT_STATUS %} +{% if grains.role not in ['so-heavynode'] %} run_installer: cmd.script: - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 - cwd: /opt/so - - args: -token={{ GRIDNODETOKEN }} + - args: -token={{ GRIDNODETOKENGENERAL }} +{% else %} +run_installer: + cmd.script: + - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 + - cwd: /opt/so + - args: -token={{ GRIDNODETOKENHEAVY }} +{% endif %} {% endif %} diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index 771d923ef..54538ed9e 100755 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -25,11 +25,30 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then fi done - # Grid Nodes - for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes/*.json + # Grid Nodes - General + for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json do - printf "\n\nGrid Nodes Policy - Loading $INTEGRATION\n" - elastic_fleet_integration_check "so-grid-nodes" "$INTEGRATION" + printf "\n\nGrid Nodes Policy_General - Loading $INTEGRATION\n" + elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION" + if [ -n "$INTEGRATION_ID" ]; then + printf "\n\nIntegration $NAME exists - Updating integration\n" + elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION" + else + printf "\n\nIntegration does not exist - Creating integration\n" + if [ "$NAME" != "elasticsearch-logs" ]; then + elastic_fleet_integration_create "@$INTEGRATION" + fi + fi + done + if [[ "$RETURN_CODE" != "1" ]]; then + touch /opt/so/state/eaintegrations.txt + fi + + # Grid Nodes - Heavy + for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json + do + printf "\n\nGrid Nodes Policy_Heavy - Loading $INTEGRATION\n" + elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION" if [ -n "$INTEGRATION_ID" ]; then printf "\n\nIntegration $NAME exists - Updating integration\n" elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION" diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 94a42a70a..3b75fef56 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -62,8 +62,11 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fle # Initial Endpoints Policy elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600" -# Grid Nodes Policy -elastic_fleet_policy_create "so-grid-nodes" "SO Grid Node Policy" "false" "1209600" +# Grid Nodes - General Policy +elastic_fleet_policy_create "so-grid-nodes_general" "SO Grid Nodes - General Purpose" "false" "1209600" + +# Grid Nodes - Heavy Node Policy +elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy Node" "false" "1209600" # Load Integrations for default policies so-elastic-fleet-integration-policy-load @@ -81,7 +84,8 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl # Query for Enrollment Tokens for default policies ENDPOINTSENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') -GRIDNODESENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes")) | .api_key') +GRIDNODESENROLLMENTOKENGENERAL=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key') +GRIDNODESENROLLMENTOKENHEAVY=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key') # Store needed data in minion pillar pillar_file=/opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls @@ -92,13 +96,15 @@ printf '%s\n'\ " server:"\ " es_token: '$ESTOKEN'"\ " endpoints_enrollment: '$ENDPOINTSENROLLMENTOKEN'"\ - " grid_enrollment: '$GRIDNODESENROLLMENTOKEN'"\ + " grid_enrollment_general: '$GRIDNODESENROLLMENTOKENGENERAL'"\ + " grid_enrollment_heavy: '$GRIDNODESENROLLMENTOKENHEAVY'"\ "" >> "$pillar_file" #Store Grid Nodes Enrollment token in Global pillar global_pillar_file=/opt/so/saltstack/local/pillar/global/soc_global.sls printf '%s\n'\ - " fleet_grid_enrollment_token: '$GRIDNODESENROLLMENTOKEN'"\ + " fleet_grid_enrollment_token_general: '$GRIDNODESENROLLMENTOKENGENERAL'"\ + " fleet_grid_enrollment_token_heavy: '$GRIDNODESENROLLMENTOKENHEAVY'"\ "" >> "$global_pillar_file" # Call Elastic-Fleet Salt State diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index d367de287..1b8728bf5 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,13 +1,23 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %} -{% from 'logstash/map.jinja' import LOGSTASH_NODES with context %} {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} +{# ES_LOGSTASH_NODES is the same as LOGSTASH_NODES from logstash/map.jinja but heavynodes are removed #} +{% set ES_LOGSTASH_NODES = [] %} +{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} +{% for node_type, node_details in node_data.items() | sort %} +{% if node_type != 'heavynode' %} +{% for hostname in node_data[node_type].keys() %} +{% do ES_LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %} +{% endfor %} +{% endif %} +{% endfor %} + {% if grains.id.split('_') | last in ['manager','managersearch','standalone'] %} - {% if LOGSTASH_NODES | length > 1 %} + {% if ES_LOGSTASH_NODES | length > 1 %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %} - {% for NODE in LOGSTASH_NODES %} + {% for NODE in ES_LOGSTASH_NODES %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %} {% endfor %} {% if grains.id.split('_') | last == 'manager' %} @@ -22,6 +32,8 @@ {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.roles.extend(['ml', 'master', 'transform']) %} {% endif %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.manager]}}) %} +{% elif grains.id.split('_') | last == 'heavynode' %} + {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client', 'ingest']}) %} {% endif %} {% if HIGHLANDER %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.xpack.ml.update({'enabled': true}) %} diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 4777e3bce..e28ca5fdf 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -8,6 +8,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} {% from 'logstash/map.jinja' import LOGSTASH_NODES %} +{% from 'elasticsearch/config.map.jinja' import ES_LOGSTASH_NODES %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} {% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} @@ -32,7 +33,7 @@ so-elasticsearch: {% endfor %} {% endif %} - environment: - {% if LOGSTASH_NODES | length == 1 %} + {% if ES_LOGSTASH_NODES | length == 1 or GLOBALS.role == 'so-heavynode' %} - discovery.type=single-node {% endif %} - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true @@ -144,6 +145,7 @@ es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: {% endfor %} {% endif %} +{% if GLOBALS.role in GLOBALS.manager_roles %} so-es-cluster-settings: cmd.run: - name: /usr/sbin/so-elasticsearch-cluster-settings @@ -152,6 +154,7 @@ so-es-cluster-settings: - require: - docker_container: so-elasticsearch - file: elasticsearch_sbin_jinja +{% endif %} so-elasticsearch-ilm-policy-load: cmd.run: diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings index 83a997845..9048b85b1 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings @@ -3,6 +3,8 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +{%- from 'vars/globals.map.jinja' import GLOBALS %} +{%- set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} . /usr/sbin/so-common @@ -14,17 +16,17 @@ COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 30 ]]; do curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://localhost:"$ELASTICSEARCH_PORT" - if [ $? -eq 0 ]; then + if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" - break + break else ((COUNT+=1)) - sleep 1 + sleep 1 echo -n "." fi done -if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then +if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then echo echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'docker ps' \n -running 'sudo so-elastic-restart'" echo @@ -32,9 +34,12 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then exit fi -# Check to see if config already exists -CLUSTER_SETTINGS=$(so-elasticsearch-query _cluster/settings | jq .persistent.cluster.remote) -if [[ ! -z "$CLUSTER_SETTINGS" ]]; then +{%- if GLOBALS.role in GLOBALS.manager_roles %} echo "Applying cross cluster search config..." so-elasticsearch-query _cluster/settings -d "{\"persistent\": {\"cluster\": {\"remote\": {\"{{ GLOBALS.manager }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" -XPUT -fi +{%- if node_data['heavynode'] is defined %} +{%- for hostname, node_details in node_data['heavynode'].items() %} + so-elasticsearch-query _cluster/settings -d "{\"persistent\": {\"cluster\": {\"remote\": {\"{{ hostname }}\": {\"seeds\": [\"{{node_details.ip}}:9300\"]}}}}}" -XPUT +{%- endfor %} +{%- endif %} +{%- endif %} diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 176d12ec2..45567de91 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -417,6 +417,14 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + receiver: + portgroups: + - yum + - docker_registry + - influxdb + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update self: portgroups: - syslog @@ -486,6 +494,9 @@ firewall: heavynode: portgroups: - salt_manager + receiver: + portgroups: + - salt_manager customhostgroup0: portgroups: [] customhostgroup1: @@ -569,6 +580,14 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + receiver: + portgroups: + - yum + - docker_registry + - influxdb + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update self: portgroups: - syslog @@ -638,6 +657,9 @@ firewall: heavynode: portgroups: - salt_manager + receiver: + portgroups: + - salt_manager customhostgroup0: portgroups: [] customhostgroup1: @@ -731,6 +753,14 @@ firewall: - redis - elasticsearch_rest - elasticsearch_node + receiver: + portgroups: + - yum + - docker_registry + - influxdb + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update self: portgroups: - syslog @@ -806,6 +836,9 @@ firewall: heavynode: portgroups: - salt_manager + receiver: + portgroups: + - salt_manager customhostgroup0: portgroups: [] customhostgroup1: @@ -1128,7 +1161,9 @@ firewall: hostgroups: sensor: portgroups: + - beats_5044 - beats_5644 + - elastic_agent_data searchnode: portgroups: - redis diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index a273476e1..8d27730b2 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -8,6 +8,7 @@ logstash: receiver: - receiver heavynode: + - manager - search searchnode: - search diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index 91433cba8..abc28cfe6 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -58,7 +58,7 @@ so-logstash: - /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro {% endif %} - {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-eval','so-fleet'] %} + {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-eval','so-fleet', 'so-heavynode', 'so-receiver'] %} - /opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.crt:/usr/share/logstash/elasticfleet-logstash.crt:ro - /opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.p8:/usr/share/logstash/elasticfleet-logstash.key:ro {% endif %} diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index 69e102e78..4098df21f 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -8,6 +8,7 @@ {% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %} {% set REDIS_NODES = [] %} +{# LOGSTASH_NODES is the same as ES_LOGSTASH_NODES from elasticsearch/config.map.jinja but heavynodes are present #} {% set LOGSTASH_NODES = [] %} {% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index e9aff9e09..df0adb73c 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -133,6 +133,15 @@ function add_elasticsearch_to_minion() { " " >> $PILLARFILE } + +# Add Elastic Agent settings to the minion file +function add_elastic_agent_to_minion() { + printf '%s\n'\ + "elasticagent:"\ + " enabled: True"\ + " " >> $PILLARFILE +} + # Add Elastic Fleet Server settings to the minion file function add_fleet_to_minion() { @@ -518,6 +527,7 @@ function createIDH() { function createHEAVYNODE() { add_elasticsearch_to_minion + add_elastic_agent_to_minion add_logstash_to_minion add_sensor_to_minion add_strelka_to_minion diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index e077d55d0..96953ffff 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -140,7 +140,7 @@ rediskeyperms: - group: 939 {% endif %} -{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet'] %} +{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet', 'so-receiver'] %} # Create cert for Elastic Fleet Host etc_elasticfleet_key: diff --git a/salt/top.sls b/salt/top.sls index 229557575..11a594f29 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -209,6 +209,7 @@ base: - suricata - zeek - elasticfleet.install_agent_grid + - elasticagent - docker_clean '*_import and G@saltversion:{{saltversion}}': diff --git a/salt/vars/receiver.map.jinja b/salt/vars/receiver.map.jinja new file mode 100644 index 000000000..964f69663 --- /dev/null +++ b/salt/vars/receiver.map.jinja @@ -0,0 +1 @@ +{% set ROLE_GLOBALS = {} %} diff --git a/setup/so-setup b/setup/so-setup index 5f96106a5..53dba8cfd 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -489,9 +489,13 @@ if ! [[ -f $install_opt_file ]]; then check_requirements "heavynode" calculate_useable_cores networking_needful + check_network_manager_conf + set_network_dev_status_list collect_mngr_hostname add_mngr_ip_to_hosts check_manager_connection + detect_cloud + whiptail_sensor_nics set_minion_info whiptail_end_settings