Merge pull request #10653 from Security-Onion-Solutions/2.4/heavynode

2.4/heavynode
This commit is contained in:
Josh Brower
2023-06-23 19:55:24 -04:00
committed by GitHub
54 changed files with 876 additions and 44 deletions

View File

@@ -46,6 +46,7 @@
'pcap',
'suricata',
'healthcheck',
'elasticagent',
'schedule',
'tcpreplay',
'docker_clean'

View File

@@ -178,6 +178,11 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
'so-elastic-agent':
final_octet: 46
custom_bind_mounts: []
extra_hosts: []
extra_env: []
'so-telegraf':
final_octet: 99
custom_bind_mounts: []

View File

@@ -0,0 +1,47 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% if sls.split('.')[0] in allowed_states %}
# Add EA Group
elasticagentgroup:
group.present:
- name: elastic-agent
- gid: 949
# Add EA user
elastic-agent:
user.present:
- uid: 949
- gid: 949
- home: /opt/so/conf/elastic-agent
- createhome: False
elasticagentconfdir:
file.directory:
- name: /opt/so/conf/elastic-agent
- user: 949
- group: 939
- makedirs: True
# Create config
create-elastic-agent-config:
file.managed:
- name: /opt/so/conf/elastic-agent/elastic-agent.yml
- source: salt://elasticagent/files/elastic-agent.yml.jinja
- user: 949
- group: 939
- template: jinja
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}

View File

@@ -0,0 +1,2 @@
elasticagent:
enabled: False

View File

@@ -0,0 +1,27 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
include:
- elasticagent.sostatus
so-elastic-agent:
docker_container.absent:
- force: True
so-elastic-agent_so-status.disabled:
file.comment:
- name: /opt/so/conf/so-status/so-status.conf
- regex: ^so-elastic-agent$
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}

View File

@@ -0,0 +1,62 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKER %}
include:
- elasticagent.config
- elasticagent.sostatus
so-elastic-agent:
docker_container.running:
- image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }}
- name: so-elastic-agent
- hostname: {{ GLOBALS.hostname }}
- detach: True
- user: 949
- networks:
- sobridge:
- ipv4_address: {{ DOCKER.containers['so-elastic-agent'].ip }}
- extra_hosts:
- {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
- {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }}
{% if DOCKER.containers['so-elastic-agent'].extra_hosts %}
{% for XTRAHOST in DOCKER.containers['so-elastic-agent'].extra_hosts %}
- {{ XTRAHOST }}
{% endfor %}
{% endif %}
- binds:
- /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro
- /nsm:/nsm:ro
{% if DOCKER.containers['so-elastic-agent'].custom_bind_mounts %}
{% for BIND in DOCKER.containers['so-elastic-agent'].custom_bind_mounts %}
- {{ BIND }}
{% endfor %}
{% endif %}
{% if DOCKER.containers['so-elastic-agent'].extra_env %}
- environment:
{% for XTRAENV in DOCKER.containers['so-elastic-agent'].extra_env %}
- {{ XTRAENV }}
{% endfor %}
{% endif %}
delete_so-elastic-agent_so-status.disabled:
file.uncomment:
- name: /opt/so/conf/so-status/so-status.conf
- regex: ^so-elastic-agent$
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}

View File

@@ -0,0 +1,119 @@
{% from 'vars/globals.map.jinja' import GLOBALS %}
{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
id: aea1ba80-1065-11ee-a369-97538913b6a9
revision: 2
outputs:
default:
type: elasticsearch
hosts:
- 'https://{{ GLOBALS.hostname }}:9200'
username: '{{ ES_USER }}'
password: '{{ ES_PASS }}'
ssl.verification_mode: none
output_permissions: {}
agent:
download:
sourceURI: 'http://{{ GLOBALS.manager }}:8443/artifacts/'
monitoring:
enabled: false
logs: false
metrics: false
features: {}
inputs:
- id: logfile-logs-80ffa884-2cfc-459a-964a-34df25714d85
name: suricata-logs
revision: 1
type: logfile
use_output: default
meta:
package:
name: log
version:
data_stream:
namespace: so
package_policy_id: 80ffa884-2cfc-459a-964a-34df25714d85
streams:
- id: logfile-log.log-80ffa884-2cfc-459a-964a-34df25714d85
data_stream:
dataset: suricata
paths:
- /nsm/suricata/eve*.json
processors:
- add_fields:
target: event
fields:
category: network
module: suricata
pipeline: suricata.common
- id: logfile-logs-90103ac4-f6bd-4a4a-b596-952c332390fc
name: strelka-logs
revision: 1
type: logfile
use_output: default
meta:
package:
name: log
version:
data_stream:
namespace: so
package_policy_id: 90103ac4-f6bd-4a4a-b596-952c332390fc
streams:
- id: logfile-log.log-90103ac4-f6bd-4a4a-b596-952c332390fc
data_stream:
dataset: strelka
paths:
- /nsm/strelka/log/strelka.log
processors:
- add_fields:
target: event
fields:
category: file
module: strelka
pipeline: strelka.file
- id: logfile-logs-6197fe84-9b58-4d9b-8464-3d517f28808d
name: zeek-logs
revision: 1
type: logfile
use_output: default
meta:
package:
name: log
version:
data_stream:
namespace: so
package_policy_id: 6197fe84-9b58-4d9b-8464-3d517f28808d
streams:
- id: logfile-log.log-6197fe84-9b58-4d9b-8464-3d517f28808d
data_stream:
dataset: zeek
paths:
- /nsm/zeek/logs/current/*.log
processors:
- dissect:
tokenizer: '/nsm/zeek/logs/current/%{pipeline}.log'
field: log.file.path
trim_chars: .log
target_prefix: ''
- script:
lang: javascript
source: |
function process(event) {
var pl = event.Get("pipeline");
event.Put("@metadata.pipeline", "zeek." + pl);
}
- add_fields:
target: event
fields:
category: network
module: zeek
- add_tags:
tags: ics
when:
regexp:
pipeline: >-
^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*
exclude_files:
- >-
broker|capture_loss|cluster|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout.log$

View File

@@ -0,0 +1,13 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'elasticagent/map.jinja' import ELASTICAGENTMERGED %}
include:
{% if ELASTICAGENTMERGED.enabled %}
- elasticagent.enabled
{% else %}
- elasticagent.disabled
{% endif %}

View File

@@ -0,0 +1,7 @@
{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
https://securityonion.net/license; you may not use this file except in compliance with the
Elastic License 2.0. #}
{% import_yaml 'elasticagent/defaults.yaml' as ELASTICAGENTDEFAULTS %}
{% set ELASTICAGENTMERGED = salt['pillar.get']('elasticagent', ELASTICAGENTDEFAULTS.elasticagent, merge=True) %}

View File

@@ -0,0 +1,21 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
append_so-elastic-agent_so-status.conf:
file.append:
- name: /opt/so/conf/so-status/so-status.conf
- text: so-elastic-agent
- unless: grep -q so-elastic-agent$ /opt/so/conf/so-status/so-status.conf
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}

View File

@@ -0,0 +1,10 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
/usr/sbin/so-restart elastic-agent $1

View File

@@ -0,0 +1,12 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
/usr/sbin/so-start elastic-agent $1

View File

@@ -0,0 +1,12 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
/usr/sbin/so-stop elastic-agent $1

View File

@@ -8,13 +8,13 @@
{% if sls.split('.')[0] in allowed_states %}
# Add EA Group
elasticsagentgroup:
elasticfleetgroup:
group.present:
- name: elastic-agent
- name: elastic-fleet
- gid: 947
# Add EA user
elastic-agent:
elastic-fleet:
user.present:
- uid: 947
- gid: 947

View File

@@ -0,0 +1,106 @@
{
"package": {
"name": "elasticsearch",
"version": ""
},
"name": "elasticsearch-logs",
"namespace": "default",
"description": "Elasticsearch Logs",
"policy_id": "so-grid-nodes_general",
"inputs": {
"elasticsearch-logfile": {
"enabled": true,
"streams": {
"elasticsearch.audit": {
"enabled": false,
"vars": {
"paths": [
"/var/log/elasticsearch/*_audit.json"
]
}
},
"elasticsearch.deprecation": {
"enabled": false,
"vars": {
"paths": [
"/var/log/elasticsearch/*_deprecation.json"
]
}
},
"elasticsearch.gc": {
"enabled": false,
"vars": {
"paths": [
"/var/log/elasticsearch/gc.log.[0-9]*",
"/var/log/elasticsearch/gc.log"
]
}
},
"elasticsearch.server": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/elasticsearch/*.log"
]
}
},
"elasticsearch.slowlog": {
"enabled": false,
"vars": {
"paths": [
"/var/log/elasticsearch/*_index_search_slowlog.json",
"/var/log/elasticsearch/*_index_indexing_slowlog.json"
]
}
}
}
},
"elasticsearch-elasticsearch/metrics": {
"enabled": false,
"vars": {
"hosts": [
"http://localhost:9200"
],
"scope": "node"
},
"streams": {
"elasticsearch.stack_monitoring.ccr": {
"enabled": false
},
"elasticsearch.stack_monitoring.cluster_stats": {
"enabled": false
},
"elasticsearch.stack_monitoring.enrich": {
"enabled": false
},
"elasticsearch.stack_monitoring.index": {
"enabled": false
},
"elasticsearch.stack_monitoring.index_recovery": {
"enabled": false,
"vars": {
"active.only": true
}
},
"elasticsearch.stack_monitoring.index_summary": {
"enabled": false
},
"elasticsearch.stack_monitoring.ml_job": {
"enabled": false
},
"elasticsearch.stack_monitoring.node": {
"enabled": false
},
"elasticsearch.stack_monitoring.node_stats": {
"enabled": false
},
"elasticsearch.stack_monitoring.pending_tasks": {
"enabled": false
},
"elasticsearch.stack_monitoring.shard": {
"enabled": false
}
}
}
}
}

View File

@@ -6,7 +6,7 @@
"name": "idh-logs",
"namespace": "so",
"description": "IDH integration",
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_general",
"inputs": {
"logs-logfile": {
"enabled": true,

View File

@@ -6,7 +6,7 @@
"name": "import-evtx-logs",
"namespace": "so",
"description": "Import Windows EVTX logs",
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_general",
"vars": {},
"inputs": {
"logs-logfile": {

View File

@@ -6,7 +6,7 @@
"name": "import-suricata-logs",
"namespace": "so",
"description": "Import Suricata logs",
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_general",
"inputs": {
"logs-logfile": {
"enabled": true,

View File

@@ -0,0 +1,29 @@
{
"package": {
"name": "log",
"version": ""
},
"name": "kratos-logs",
"namespace": "so",
"description": "Kratos logs",
"policy_id": "so-grid-nodes_general",
"inputs": {
"logs-logfile": {
"enabled": true,
"streams": {
"log.log": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/kratos/kratos.log"
],
"data_stream.dataset": "kratos",
"tags": ["so-kratos"],
"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos",
"custom": "pipeline: kratos"
}
}
}
}
}
}

View File

@@ -0,0 +1,20 @@
{
"package": {
"name": "osquery_manager",
"version": ""
},
"name": "osquery-grid-nodes",
"namespace": "default",
"policy_id": "so-grid-nodes_general",
"inputs": {
"osquery_manager-osquery": {
"enabled": true,
"streams": {
"osquery_manager.result": {
"enabled": true,
"vars": {}
}
}
}
}
}

View File

@@ -0,0 +1,76 @@
{
"package": {
"name": "redis",
"version": ""
},
"name": "redis-logs",
"namespace": "default",
"description": "Redis logs",
"policy_id": "so-grid-nodes_general",
"inputs": {
"redis-logfile": {
"enabled": true,
"streams": {
"redis.log": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/redis/redis.log"
],
"tags": [
"redis-log"
],
"preserve_original_event": false
}
}
}
},
"redis-redis": {
"enabled": false,
"streams": {
"redis.slowlog": {
"enabled": false,
"vars": {
"hosts": [
"127.0.0.1:6379"
],
"password": ""
}
}
}
},
"redis-redis/metrics": {
"enabled": false,
"vars": {
"hosts": [
"127.0.0.1:6379"
],
"idle_timeout": "20s",
"maxconn": 10,
"network": "tcp",
"password": ""
},
"streams": {
"redis.info": {
"enabled": false,
"vars": {
"period": "10s"
}
},
"redis.key": {
"enabled": false,
"vars": {
"key.patterns": "- limit: 20\n pattern: *\n",
"period": "10s"
}
},
"redis.keyspace": {
"enabled": false,
"vars": {
"period": "10s"
}
}
}
}
}
}

View File

@@ -0,0 +1,29 @@
{
"package": {
"name": "log",
"version": ""
},
"name": "soc-auth-sync-logs",
"namespace": "so",
"description": "Security Onion - Elastic Auth Sync - Logs",
"policy_id": "so-grid-nodes_general",
"inputs": {
"logs-logfile": {
"enabled": true,
"streams": {
"log.log": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/soc/sync.log"
],
"data_stream.dataset": "soc",
"tags": ["so-soc"],
"processors": "- dissect:\n tokenizer: \"%{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: auth_sync",
"custom": "pipeline: common"
}
}
}
}
}
}

View File

@@ -0,0 +1,29 @@
{
"package": {
"name": "log",
"version": ""
},
"name": "soc-salt-relay-logs",
"namespace": "so",
"description": "Security Onion - Salt Relay - Logs",
"policy_id": "so-grid-nodes_general",
"inputs": {
"logs-logfile": {
"enabled": true,
"streams": {
"log.log": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/soc/salt-relay.log"
],
"data_stream.dataset": "soc",
"tags": ["so-soc"],
"processors": "- dissect:\n tokenizer: \"%{soc.ts} | %{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: salt_relay",
"custom": "pipeline: common"
}
}
}
}
}
}

View File

@@ -0,0 +1,29 @@
{
"package": {
"name": "log",
"version": ""
},
"name": "soc-sensoroni-logs",
"namespace": "so",
"description": "Security Onion - Sensoroni - Logs",
"policy_id": "so-grid-nodes_general",
"inputs": {
"logs-logfile": {
"enabled": true,
"streams": {
"log.log": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/sensoroni/sensoroni.log"
],
"data_stream.dataset": "soc",
"tags": [],
"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"sensoroni\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: sensoroni\n- rename:\n fields:\n - from: \"sensoroni.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"sensoroni.fields.status\"\n to: \"http.response.status_code\"\n - from: \"sensoroni.fields.method\"\n to: \"http.request.method\"\n - from: \"sensoroni.fields.path\"\n to: \"url.path\"\n - from: \"sensoroni.message\"\n to: \"event.action\"\n - from: \"sensoroni.level\"\n to: \"log.level\"\n ignore_missing: true",
"custom": "pipeline: common"
}
}
}
}
}
}

View File

@@ -0,0 +1,29 @@
{
"package": {
"name": "log",
"version": ""
},
"name": "soc-server-logs",
"namespace": "so",
"description": "Security Onion Console Logs",
"policy_id": "so-grid-nodes_general",
"inputs": {
"logs-logfile": {
"enabled": true,
"streams": {
"log.log": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/soc/sensoroni-server.log"
],
"data_stream.dataset": "soc",
"tags": ["so-soc"],
"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"soc\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: server\n- rename:\n fields:\n - from: \"soc.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"soc.fields.status\"\n to: \"http.response.status_code\"\n - from: \"soc.fields.method\"\n to: \"http.request.method\"\n - from: \"soc.fields.path\"\n to: \"url.path\"\n - from: \"soc.message\"\n to: \"event.action\"\n - from: \"soc.level\"\n to: \"log.level\"\n ignore_missing: true",
"custom": "pipeline: common"
}
}
}
}
}
}

View File

@@ -6,7 +6,7 @@
"name": "strelka-logs",
"namespace": "so",
"description": "Strelka logs",
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_general",
"inputs": {
"logs-logfile": {
"enabled": true,

View File

@@ -6,7 +6,7 @@
"name": "suricata-logs",
"namespace": "so",
"description": "Suricata integration",
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_general",
"inputs": {
"logs-logfile": {
"enabled": true,

View File

@@ -6,7 +6,7 @@
"name": "syslog-tcp-514",
"namespace": "so",
"description": "Syslog Over TCP Port 514",
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_general",
"inputs": {
"tcp-tcp": {
"enabled": true,

View File

@@ -6,7 +6,7 @@
"name": "syslog-udp-514",
"namespace": "so",
"description": "Syslog over UDP Port 514",
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_general",
"inputs": {
"udp-udp": {
"enabled": true,

View File

@@ -0,0 +1,40 @@
{
"policy_id": "so-grid-nodes_general",
"package": {
"name": "system",
"version": ""
},
"name": "system-grid-nodes",
"namespace": "default",
"inputs": {
"system-logfile": {
"enabled": true,
"streams": {
"system.auth": {
"enabled": true,
"vars": {
"paths": [
"/var/log/auth.log*",
"/var/log/secure*"
]
}
},
"system.syslog": {
"enabled": true,
"vars": {
"paths": [
"/var/log/messages*",
"/var/log/syslog*"
]
}
}
}
},
"system-winlog": {
"enabled": false
},
"system-system/metrics": {
"enabled": false
}
}
}

View File

@@ -6,7 +6,7 @@
"name": "elasticsearch-logs",
"namespace": "default",
"description": "Elasticsearch Logs",
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_heavy",
"inputs": {
"elasticsearch-logfile": {
"enabled": true,

View File

@@ -6,7 +6,7 @@
"name": "kratos-logs",
"namespace": "so",
"description": "Kratos logs",
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_heavy",
"inputs": {
"logs-logfile": {
"enabled": true,

View File

@@ -5,7 +5,7 @@
},
"name": "osquery-grid-nodes",
"namespace": "default",
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_heavy",
"inputs": {
"osquery_manager-osquery": {
"enabled": true,

View File

@@ -6,7 +6,7 @@
"name": "redis-logs",
"namespace": "default",
"description": "Redis logs",
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_heavy",
"inputs": {
"redis-logfile": {
"enabled": true,

View File

@@ -6,7 +6,7 @@
"name": "soc-auth-sync-logs",
"namespace": "so",
"description": "Security Onion - Elastic Auth Sync - Logs",
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_heavy",
"inputs": {
"logs-logfile": {
"enabled": true,

View File

@@ -6,7 +6,7 @@
"name": "soc-salt-relay-logs",
"namespace": "so",
"description": "Security Onion - Salt Relay - Logs",
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_heavy",
"inputs": {
"logs-logfile": {
"enabled": true,

View File

@@ -6,7 +6,7 @@
"name": "soc-sensoroni-logs",
"namespace": "so",
"description": "Security Onion - Sensoroni - Logs",
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_heavy",
"inputs": {
"logs-logfile": {
"enabled": true,

View File

@@ -6,7 +6,7 @@
"name": "soc-server-logs",
"namespace": "so",
"description": "Security Onion Console Logs",
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_heavy",
"inputs": {
"logs-logfile": {
"enabled": true,

View File

@@ -1,5 +1,5 @@
{
"policy_id": "so-grid-nodes",
"policy_id": "so-grid-nodes_heavy",
"package": {
"name": "system",
"version": ""

View File

@@ -2,15 +2,24 @@
# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
# this file except in compliance with the Elastic License 2.0.
{%- set GRIDNODETOKEN = salt['pillar.get']('global:fleet_grid_enrollment_token') -%}
{%- set GRIDNODETOKENGENERAL = salt['pillar.get']('global:fleet_grid_enrollment_token_general') -%}
{%- set GRIDNODETOKENHEAVY = salt['pillar.get']('global:fleet_grid_enrollment_token_heavy') -%}
{% set AGENT_STATUS = salt['service.available']('elastic-agent') %}
{% if not AGENT_STATUS %}
{% if grains.role not in ['so-heavynode'] %}
run_installer:
cmd.script:
- name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64
- cwd: /opt/so
- args: -token={{ GRIDNODETOKEN }}
- args: -token={{ GRIDNODETOKENGENERAL }}
{% else %}
run_installer:
cmd.script:
- name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64
- cwd: /opt/so
- args: -token={{ GRIDNODETOKENHEAVY }}
{% endif %}
{% endif %}

View File

@@ -25,11 +25,30 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
fi
done
# Grid Nodes
for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes/*.json
# Grid Nodes - General
for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json
do
printf "\n\nGrid Nodes Policy - Loading $INTEGRATION\n"
elastic_fleet_integration_check "so-grid-nodes" "$INTEGRATION"
printf "\n\nGrid Nodes Policy_General - Loading $INTEGRATION\n"
elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION"
if [ -n "$INTEGRATION_ID" ]; then
printf "\n\nIntegration $NAME exists - Updating integration\n"
elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
else
printf "\n\nIntegration does not exist - Creating integration\n"
if [ "$NAME" != "elasticsearch-logs" ]; then
elastic_fleet_integration_create "@$INTEGRATION"
fi
fi
done
if [[ "$RETURN_CODE" != "1" ]]; then
touch /opt/so/state/eaintegrations.txt
fi
# Grid Nodes - Heavy
for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json
do
printf "\n\nGrid Nodes Policy_Heavy - Loading $INTEGRATION\n"
elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION"
if [ -n "$INTEGRATION_ID" ]; then
printf "\n\nIntegration $NAME exists - Updating integration\n"
elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"

View File

@@ -62,8 +62,11 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fle
# Initial Endpoints Policy
elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600"
# Grid Nodes Policy
elastic_fleet_policy_create "so-grid-nodes" "SO Grid Node Policy" "false" "1209600"
# Grid Nodes - General Policy
elastic_fleet_policy_create "so-grid-nodes_general" "SO Grid Nodes - General Purpose" "false" "1209600"
# Grid Nodes - Heavy Node Policy
elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy Node" "false" "1209600"
# Load Integrations for default policies
so-elastic-fleet-integration-policy-load
@@ -81,7 +84,8 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl
# Query for Enrollment Tokens for default policies
ENDPOINTSENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
GRIDNODESENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes")) | .api_key')
GRIDNODESENROLLMENTOKENGENERAL=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key')
GRIDNODESENROLLMENTOKENHEAVY=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key')
# Store needed data in minion pillar
pillar_file=/opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls
@@ -92,13 +96,15 @@ printf '%s\n'\
" server:"\
" es_token: '$ESTOKEN'"\
" endpoints_enrollment: '$ENDPOINTSENROLLMENTOKEN'"\
" grid_enrollment: '$GRIDNODESENROLLMENTOKEN'"\
" grid_enrollment_general: '$GRIDNODESENROLLMENTOKENGENERAL'"\
" grid_enrollment_heavy: '$GRIDNODESENROLLMENTOKENHEAVY'"\
"" >> "$pillar_file"
#Store Grid Nodes Enrollment token in Global pillar
global_pillar_file=/opt/so/saltstack/local/pillar/global/soc_global.sls
printf '%s\n'\
" fleet_grid_enrollment_token: '$GRIDNODESENROLLMENTOKEN'"\
" fleet_grid_enrollment_token_general: '$GRIDNODESENROLLMENTOKENGENERAL'"\
" fleet_grid_enrollment_token_heavy: '$GRIDNODESENROLLMENTOKENHEAVY'"\
"" >> "$global_pillar_file"
# Call Elastic-Fleet Salt State

View File

@@ -1,13 +1,23 @@
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %}
{% from 'logstash/map.jinja' import LOGSTASH_NODES with context %}
{% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
{# ES_LOGSTASH_NODES is the same as LOGSTASH_NODES from logstash/map.jinja but heavynodes are removed #}
{% set ES_LOGSTASH_NODES = [] %}
{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
{% for node_type, node_details in node_data.items() | sort %}
{% if node_type != 'heavynode' %}
{% for hostname in node_data[node_type].keys() %}
{% do ES_LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %}
{% endfor %}
{% endif %}
{% endfor %}
{% if grains.id.split('_') | last in ['manager','managersearch','standalone'] %}
{% if LOGSTASH_NODES | length > 1 %}
{% if ES_LOGSTASH_NODES | length > 1 %}
{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %}
{% for NODE in LOGSTASH_NODES %}
{% for NODE in ES_LOGSTASH_NODES %}
{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %}
{% endfor %}
{% if grains.id.split('_') | last == 'manager' %}
@@ -22,6 +32,8 @@
{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.roles.extend(['ml', 'master', 'transform']) %}
{% endif %}
{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.manager]}}) %}
{% elif grains.id.split('_') | last == 'heavynode' %}
{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client', 'ingest']}) %}
{% endif %}
{% if HIGHLANDER %}
{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.xpack.ml.update({'enabled': true}) %}

View File

@@ -8,6 +8,7 @@
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKER %}
{% from 'logstash/map.jinja' import LOGSTASH_NODES %}
{% from 'elasticsearch/config.map.jinja' import ES_LOGSTASH_NODES %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
@@ -32,7 +33,7 @@ so-elasticsearch:
{% endfor %}
{% endif %}
- environment:
{% if LOGSTASH_NODES | length == 1 %}
{% if ES_LOGSTASH_NODES | length == 1 or GLOBALS.role == 'so-heavynode' %}
- discovery.type=single-node
{% endif %}
- ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true
@@ -144,6 +145,7 @@ es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
{% endfor %}
{% endif %}
{% if GLOBALS.role in GLOBALS.manager_roles %}
so-es-cluster-settings:
cmd.run:
- name: /usr/sbin/so-elasticsearch-cluster-settings
@@ -152,6 +154,7 @@ so-es-cluster-settings:
- require:
- docker_container: so-elasticsearch
- file: elasticsearch_sbin_jinja
{% endif %}
so-elasticsearch-ilm-policy-load:
cmd.run:

View File

@@ -3,6 +3,8 @@
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- from 'vars/globals.map.jinja' import GLOBALS %}
{%- set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
. /usr/sbin/so-common
@@ -14,17 +16,17 @@ COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 30 ]]; do
curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://localhost:"$ELASTICSEARCH_PORT"
if [ $? -eq 0 ]; then
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
break
break
else
((COUNT+=1))
sleep 1
sleep 1
echo -n "."
fi
done
if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
echo
echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'docker ps' \n -running 'sudo so-elastic-restart'"
echo
@@ -32,9 +34,12 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
exit
fi
# Check to see if config already exists
CLUSTER_SETTINGS=$(so-elasticsearch-query _cluster/settings | jq .persistent.cluster.remote)
if [[ ! -z "$CLUSTER_SETTINGS" ]]; then
{%- if GLOBALS.role in GLOBALS.manager_roles %}
echo "Applying cross cluster search config..."
so-elasticsearch-query _cluster/settings -d "{\"persistent\": {\"cluster\": {\"remote\": {\"{{ GLOBALS.manager }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" -XPUT
fi
{%- if node_data['heavynode'] is defined %}
{%- for hostname, node_details in node_data['heavynode'].items() %}
so-elasticsearch-query _cluster/settings -d "{\"persistent\": {\"cluster\": {\"remote\": {\"{{ hostname }}\": {\"seeds\": [\"{{node_details.ip}}:9300\"]}}}}}" -XPUT
{%- endfor %}
{%- endif %}
{%- endif %}

View File

@@ -417,6 +417,14 @@ firewall:
- elastic_agent_control
- elastic_agent_data
- elastic_agent_update
receiver:
portgroups:
- yum
- docker_registry
- influxdb
- elastic_agent_control
- elastic_agent_data
- elastic_agent_update
self:
portgroups:
- syslog
@@ -486,6 +494,9 @@ firewall:
heavynode:
portgroups:
- salt_manager
receiver:
portgroups:
- salt_manager
customhostgroup0:
portgroups: []
customhostgroup1:
@@ -569,6 +580,14 @@ firewall:
- elastic_agent_control
- elastic_agent_data
- elastic_agent_update
receiver:
portgroups:
- yum
- docker_registry
- influxdb
- elastic_agent_control
- elastic_agent_data
- elastic_agent_update
self:
portgroups:
- syslog
@@ -638,6 +657,9 @@ firewall:
heavynode:
portgroups:
- salt_manager
receiver:
portgroups:
- salt_manager
customhostgroup0:
portgroups: []
customhostgroup1:
@@ -731,6 +753,14 @@ firewall:
- redis
- elasticsearch_rest
- elasticsearch_node
receiver:
portgroups:
- yum
- docker_registry
- influxdb
- elastic_agent_control
- elastic_agent_data
- elastic_agent_update
self:
portgroups:
- syslog
@@ -806,6 +836,9 @@ firewall:
heavynode:
portgroups:
- salt_manager
receiver:
portgroups:
- salt_manager
customhostgroup0:
portgroups: []
customhostgroup1:
@@ -1128,7 +1161,9 @@ firewall:
hostgroups:
sensor:
portgroups:
- beats_5044
- beats_5644
- elastic_agent_data
searchnode:
portgroups:
- redis

View File

@@ -8,6 +8,7 @@ logstash:
receiver:
- receiver
heavynode:
- manager
- search
searchnode:
- search

View File

@@ -58,7 +58,7 @@ so-logstash:
- /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro
- /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro
{% endif %}
{% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-eval','so-fleet'] %}
{% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-eval','so-fleet', 'so-heavynode', 'so-receiver'] %}
- /opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.crt:/usr/share/logstash/elasticfleet-logstash.crt:ro
- /opt/so/conf/elastic-fleet/certs/elasticfleet-logstash.p8:/usr/share/logstash/elasticfleet-logstash.key:ro
{% endif %}

View File

@@ -8,6 +8,7 @@
{% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %}
{% set REDIS_NODES = [] %}
{# LOGSTASH_NODES is the same as ES_LOGSTASH_NODES from elasticsearch/config.map.jinja but heavynodes are present #}
{% set LOGSTASH_NODES = [] %}
{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}

View File

@@ -133,6 +133,15 @@ function add_elasticsearch_to_minion() {
" " >> $PILLARFILE
}
# Add Elastic Agent settings to the minion file
function add_elastic_agent_to_minion() {
printf '%s\n'\
"elasticagent:"\
" enabled: True"\
" " >> $PILLARFILE
}
# Add Elastic Fleet Server settings to the minion file
function add_fleet_to_minion() {
@@ -518,6 +527,7 @@ function createIDH() {
function createHEAVYNODE() {
add_elasticsearch_to_minion
add_elastic_agent_to_minion
add_logstash_to_minion
add_sensor_to_minion
add_strelka_to_minion

View File

@@ -140,7 +140,7 @@ rediskeyperms:
- group: 939
{% endif %}
{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet'] %}
{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet', 'so-receiver'] %}
# Create cert for Elastic Fleet Host
etc_elasticfleet_key:

View File

@@ -209,6 +209,7 @@ base:
- suricata
- zeek
- elasticfleet.install_agent_grid
- elasticagent
- docker_clean
'*_import and G@saltversion:{{saltversion}}':

View File

@@ -0,0 +1 @@
{% set ROLE_GLOBALS = {} %}

View File

@@ -489,9 +489,13 @@ if ! [[ -f $install_opt_file ]]; then
check_requirements "heavynode"
calculate_useable_cores
networking_needful
check_network_manager_conf
set_network_dev_status_list
collect_mngr_hostname
add_mngr_ip_to_hosts
check_manager_connection
detect_cloud
whiptail_sensor_nics
set_minion_info
whiptail_end_settings