CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

create and use redis:nodes and elasticsearch:nodes pillars

Browse Source
This commit is contained in:
m0duspwnens
2024-06-20 16:42:11 -04:00
parent 55f8303dc2
commit 81fcd68e9b
11 changed files with 114 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
pillar/elasticsearch/nodes.sls Normal file
View File

@@ -0,0 +1,34 @@
{% set node_types = {} %}
{% for minionid, ip in salt.saltutil.runner(
'mine.get',
tgt='elasticsearch:enabled:true',
fun='network.ip_addrs',
tgt_type='pillar') | dictsort()
%}
# only add a node to the pillar if it returned an ip from the mine
{% if ip | length > 0%}
{% set hostname = minionid.split('_') | first %}
{% set node_type = minionid.split('_') | last %}
{% if node_type not in node_types.keys() %}
{% do node_types.update({node_type: {hostname: ip[0]}}) %}
{% else %}
{% if hostname not in node_types[node_type] %}
{% do node_types[node_type].update({hostname: ip[0]}) %}
{% else %}
{% do node_types[node_type][hostname].update(ip[0]) %}
{% endif %}
{% endif %}
{% endif %}
{% endfor %}
elasticsearch:
nodes:
{% for node_type, values in node_types.items() %}
{{node_type}}:
{% for hostname, ip in values.items() %}
{{hostname}}:
ip: {{ip}}
{% endfor %}
{% endfor %}

9
pillar/logstash/nodes.sls
View File

@@ -1,16 +1,15 @@
{% set node_types = {} %}
{% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %}
{% for minionid, ip in salt.saltutil.runner(
'mine.get',
tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet ',
tgt='logstash:enabled:true',
fun='network.ip_addrs',
tgt_type='compound') | dictsort()
tgt_type='pillar') | dictsort()
%}
# only add a node to the pillar if it returned an ip from the mine
{% if ip | length > 0%}
{% set hostname = cached_grains[minionid]['host'] %}
{% set node_type = minionid.split('_')[1] %}
{% set hostname = minionid.split('_') | first %}
{% set node_type = minionid.split('_') | last %}
{% if node_type not in node_types.keys() %}
{% do node_types.update({node_type: {hostname: ip[0]}}) %}
{% else %}

34
pillar/redis/nodes.sls Normal file
View File

@@ -0,0 +1,34 @@
{% set node_types = {} %}
{% for minionid, ip in salt.saltutil.runner(
'mine.get',
tgt='redis:enabled:true',
fun='network.ip_addrs',
tgt_type='pillar') | dictsort()
%}
# only add a node to the pillar if it returned an ip from the mine
{% if ip | length > 0%}
{% set hostname = minionid.split('_') | first %}
{% set node_type = minionid.split('_') | last %}
{% if node_type not in node_types.keys() %}
{% do node_types.update({node_type: {hostname: ip[0]}}) %}
{% else %}
{% if hostname not in node_types[node_type] %}
{% do node_types[node_type].update({hostname: ip[0]}) %}
{% else %}
{% do node_types[node_type][hostname].update(ip[0]) %}
{% endif %}
{% endif %}
{% endif %}
{% endfor %}
redis:
nodes:
{% for node_type, values in node_types.items() %}
{{node_type}}:
{% for hostname, ip in values.items() %}
{{hostname}}:
ip: {{ip}}
{% endfor %}
{% endfor %}

6
pillar/top.sls
View File

@@ -47,10 +47,12 @@ base:
- kibana.adv_kibana
- kratos.soc_kratos
- kratos.adv_kratos
- redis.nodes
- redis.soc_redis
- redis.adv_redis
- influxdb.soc_influxdb
- influxdb.adv_influxdb
- elasticsearch.nodes
- elasticsearch.soc_elasticsearch
- elasticsearch.adv_elasticsearch
- elasticfleet.soc_elasticfleet
@@ -144,10 +146,12 @@ base:
- idstools.adv_idstools
- kratos.soc_kratos
- kratos.adv_kratos
- redis.nodes
- redis.soc_redis
- redis.adv_redis
- influxdb.soc_influxdb
- influxdb.adv_influxdb
- elasticsearch.nodes
- elasticsearch.soc_elasticsearch
- elasticsearch.adv_elasticsearch
- elasticfleet.soc_elasticfleet
@@ -209,11 +213,13 @@ base:
- logstash.nodes
- logstash.soc_logstash
- logstash.adv_logstash
- elasticsearch.nodes
- elasticsearch.soc_elasticsearch
- elasticsearch.adv_elasticsearch
{% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
- elasticsearch.auth
{% endif %}
- redis.nodes
- redis.soc_redis
- redis.adv_redis
- minions.{{ grains.id }}

23
salt/elasticsearch/config.map.jinja
View File

@@ -3,21 +3,30 @@
{% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
{# ES_LOGSTASH_NODES is the same as LOGSTASH_NODES from logstash/map.jinja but heavynodes and fleet nodes are removed #}
{% set ES_LOGSTASH_NODES = [] %}
{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
{# this is a list of dicts containing hostname:ip for elasticsearch nodes that need to know about each other for cluster #}
{% set ELASTICSEARCH_SEED_HOSTS = [] %}
{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
{% for node_type, node_details in node_data.items() | sort %}
{% if node_type not in ['heavynode', 'fleet'] %}
{% if node_type != 'heavynode' %}
{% for hostname in node_data[node_type].keys() %}
{% do ES_LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %}
{% do ELASTICSEARCH_SEED_HOSTS.append({hostname:node_details[hostname].ip}) %}
{% endfor %}
{% endif %}
{% endfor %}
{# this is a list of dicts containing hostname:ip of all nodes running elasticsearch #}
{% set ELASTICSEARCH_NODES = [] %}
{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
{% for node_type, node_details in node_data.items() %}
{% for hostname in node_data[node_type].keys() %}
{% do ELASTICSEARCH_NODES.append({hostname:node_details[hostname].ip}) %}
{% endfor %}
{% endfor %}
{% if grains.id.split('_') | last in ['manager','managersearch','standalone'] %}
{% if ES_LOGSTASH_NODES | length > 1 %}
{% if ELASTICSEARCH_SEED_HOSTS | length > 1 %}
{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %}
{% for NODE in ES_LOGSTASH_NODES %}
{% for NODE in ELASTICSEARCH_SEED_HOSTS %}
{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %}
{% endfor %}
{% endif %}

8
salt/elasticsearch/enabled.sls
View File

@@ -7,8 +7,8 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKER %}
{% from 'logstash/map.jinja' import LOGSTASH_NODES %}
{% from 'elasticsearch/config.map.jinja' import ES_LOGSTASH_NODES %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
@@ -27,7 +27,7 @@ so-elasticsearch:
- sobridge:
- ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }}
- extra_hosts:
{% for node in LOGSTASH_NODES %}
{% for node in ELASTICSEARCH_NODES %}
{% for hostname, ip in node.items() %}
- {{hostname}}:{{ip}}
{% endfor %}
@@ -38,7 +38,7 @@ so-elasticsearch:
{% endfor %}
{% endif %}
- environment:
{% if ES_LOGSTASH_NODES | length == 1 or GLOBALS.role == 'so-heavynode' %}
{% if ELASTICSEARCH_SEED_HOSTS | length == 1 or GLOBALS.role == 'so-heavynode' %}
- discovery.type=single-node
{% endif %}
- ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true

2
salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings
View File

@@ -4,7 +4,7 @@
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- from 'vars/globals.map.jinja' import GLOBALS %}
{%- set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
{%- set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
. /usr/sbin/so-common

17
salt/logstash/map.jinja
View File

@@ -7,23 +7,26 @@
{% import_yaml 'logstash/defaults.yaml' as LOGSTASH_DEFAULTS %}
{% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %}
{% set REDIS_NODES = [] %}
{# LOGSTASH_NODES is the same as ES_LOGSTASH_NODES from elasticsearch/config.map.jinja but heavynodes are present #}
{# used to store the redis nodes that logstash needs to know about to pull from the queue #}
{% set LOGSTASH_REDIS_NODES = [] %}
{# stores all logstash nodes #}
{% set LOGSTASH_NODES = [] %}
{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
{% set logstash_node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
{% set redis_node_data = salt['pillar.get']('redis:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
{% for node_type, node_details in node_data.items() | sort %}
{% for node_type, node_details in redis_node_data.items() | sort %}
{% if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch', 'so-fleet'] %}
{% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
{% for hostname in node_data[node_type].keys() %}
{% do REDIS_NODES.append({hostname:node_details[hostname].ip}) %}
{% do LOGSTASH_REDIS_NODES.append({hostname:node_details[hostname].ip}) %}
{% endfor %}
{% endif %}
{% else %}
{% do REDIS_NODES.append({GLOBALS.hostname:GLOBALS.node_ip}) %}
{% endif %}
{% endfor %}
{% for node_type, node_details in logstash_node_data.items() | sort %}
{% for hostname in node_data[node_type].keys() %}
{% do LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %}
{% endfor %}
{% endfor %}

6
salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja
View File

@@ -1,8 +1,8 @@
{%- from 'logstash/map.jinja' import REDIS_NODES with context %}
{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES with context %}
{%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
{%- for index in range(REDIS_NODES|length) %}
{%- for host in REDIS_NODES[index] %}
{%- for index in range(LOGSTASH_REDIS_NODES|length) %}
{%- for host in LOGSTASH_REDIS_NODES[index] %}
input {
redis {
host => '{{ host }}'

2
salt/soc/defaults.map.jinja
View File

@@ -14,7 +14,7 @@
{% endfor %}
{# add all grid heavy nodes to soc.server.modules.elastic.remoteHostUrls #}
{% for node_type, minions in salt['pillar.get']('logstash:nodes', {}).items() %}
{% for node_type, minions in salt['pillar.get']('elasticsearch:nodes', {}).items() %}
{% if node_type in ['heavynode'] %}
{% for m in minions.keys() %}
{% do SOCDEFAULTS.soc.config.server.modules.elastic.remoteHostUrls.append('https://' ~ m ~ ':9200') %}

3
salt/soc/merged.map.jinja
View File

@@ -5,9 +5,8 @@
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %}
{% from 'logstash/map.jinja' import LOGSTASH_NODES %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES as DOCKER_EXTRA_HOSTS %}
{% from 'manager/map.jinja' import MANAGERMERGED %}
{% set DOCKER_EXTRA_HOSTS = LOGSTASH_NODES %}
{% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %}
{% set SOCMERGED = salt['pillar.get']('soc', SOCDEFAULTS, merge=true) %}