CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Move In Day

Browse Source
This commit is contained in:
Mike Reeves
2022-09-07 09:06:25 -04:00
parent dcb7b49dbe
commit 2bd9dd80e2
611 changed files with 8015 additions and 16211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
salt/elasticsearch/auth.map.jinja
View File

@@ -1,7 +0,0 @@
{% set ELASTICAUTH = salt['pillar.filter_by']({
True: {
'user': salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user'),
'pass': salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass'),
'elasticcurl':'curl -K /opt/so/conf/elasticsearch/curl.config' },
False: {'elasticcurl': 'curl'},
}, pillar='elasticsearch:auth:enabled', default=False) %}

4
salt/elasticsearch/auth.sls
View File

@@ -11,8 +11,7 @@
{% set so_logstash_user_pass = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', salt['random.get_str'](72, chars=CHARS)) %}
{% set so_beats_user_pass = salt['pillar.get']('elasticsearch:auth:users:so_beats_user:pass', salt['random.get_str'](72, chars=CHARS)) %}
{% set so_monitor_user_pass = salt['pillar.get']('elasticsearch:auth:users:so_monitor_user:pass', salt['random.get_str'](72, chars=CHARS)) %}
{% set auth_enabled = salt['pillar.get']('elasticsearch:auth:enabled', False) %}
elastic_auth_pillar:
file.managed:
- name: /opt/so/saltstack/local/pillar/elasticsearch/auth.sls
@@ -21,7 +20,6 @@ elastic_auth_pillar:
- contents: |
elasticsearch:
auth:
enabled: {{ auth_enabled }}
users:
so_elastic_user:
user: so_elastic

48
salt/elasticsearch/config.map.jinja
View File

@@ -1,36 +1,32 @@
{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %}
{% from 'logstash/map.jinja' import REDIS_NODES with context %}
{% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
{% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %}
{% do ESCONFIG.elasticsearch.config.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %}
{% endif %}
{% if salt['pillar.get']('elasticsearch:true_cluster', False) %}
{% if grains.id.split('_') | last in ['manager','managersearch'] %}
{% if salt['pillar.get']('nodestab', {}) %}
{% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %}
{% if HIGHLANDER %}
{% do ESCONFIG.elasticsearch.config.node.roles.extend(['ml', 'transform']) %}
{% endif %}
{% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %}
{% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
{% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(SN.split('_')|first) %}
{% endfor %}
{% endif %}
{% if grains.id.split('_') | last == 'manager' %}
{% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': ''}) %}
{% endif %}
{% else %}
{% do ESCONFIG.elasticsearch.config.node.update({'roles': ['data', 'ingest']}) %}
{% if grains.id.split('_') | last in ['manager','managersearch'] %}
{% if REDIS_NODES | length > 1 %}
{% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %}
{% if HIGHLANDER %}
{% do ESCONFIG.elasticsearch.config.node.roles.extend(['ml', 'master', 'transform']) %}
{% do ESCONFIG.elasticsearch.config.node.roles.extend(['ml', 'transform']) %}
{% endif %}
{% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': 'hot'}) %}
{% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %}
{% for SN in REDIS_NODES.keys() %}
{% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(SN) %}
{% endfor %}
{% endif %}
{% if HIGHLANDER %}
{% do ESCONFIG.elasticsearch.config.xpack.ml.update({'enabled': true}) %}
{% endif %}
{% if grains.id.split('_') | last == 'manager' %}
{% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': ''}) %}
{% endif %}
{% elif grains.id.split('_') | last not in ['eval', 'standalone', 'import'] %}
{% do ESCONFIG.elasticsearch.config.node.update({'roles': ['data', 'ingest']}) %}
{% if HIGHLANDER %}
{% do ESCONFIG.elasticsearch.config.node.roles.extend(['ml', 'master', 'transform']) %}
{% endif %}
{% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': 'hot'}) %}
{% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %}
{% endif %}
{% if HIGHLANDER %}
{% do ESCONFIG.elasticsearch.config.xpack.ml.update({'enabled': true}) %}
{% endif %}
{# merge with the elasticsearch pillar #}

80
salt/elasticsearch/init.sls
View File

@@ -1,53 +1,27 @@
# Copyright 2014-2022 Security Onion Solutions, LLC
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls in allowed_states %}
include:
- ssl
{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
{% set MANAGER = salt['grains.get']('master') %}
{% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
{% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
{% set MANAGERIP = salt['pillar.get']('global:managerip') %}
{% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone', 'so-import'] %}
{% set esclustername = salt['pillar.get']('manager:esclustername') %}
{% set esheap = salt['pillar.get']('manager:esheap') %}
{% set ismanager = True %}
{% elif grains['role'] in ['so-node','so-heavynode'] %}
{% set esclustername = salt['pillar.get']('elasticsearch:esclustername') %}
{% set esheap = salt['pillar.get']('elasticsearch:esheap') %}
{% set ismanager = False %}
{% elif grains['role'] == 'so-helix' %}
{% set ismanager = True %} {# Solely for the sake of running so-catrust #}
{% endif %}
{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
{% set ROLES = salt['pillar.get']('elasticsearch:roles', {}) %}
{% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %}
{% from 'elasticsearch/config.map.jinja' import ESCONFIG with context %}
{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS without context %}
{% from 'logstash/map.jinja' import REDIS_NODES with context %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
vm.max_map_count:
sysctl.present:
- value: 262144
{% if ismanager %}
{% if GLOBALS.is_manager %}
# We have to add the Manager CA to the CA list
cascriptsync:
file.managed:
@@ -75,10 +49,6 @@ es_sync_scripts:
- file_mode: 755
- template: jinja
- source: salt://elasticsearch/tools/sbin
- defaults:
ELASTICCURL: 'curl'
- context:
ELASTICCURL: {{ ELASTICAUTH.elasticcurl }}
- exclude_pat:
- so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
@@ -90,8 +60,6 @@ so-elasticsearch-pipelines-script:
- group: 939
- mode: 754
- template: jinja
- defaults:
ELASTICCURL: {{ ELASTICAUTH.elasticcurl }}
# Move our new CA over so Elastic and Logstash can use SSL with the internal CA
catrustdir:
@@ -115,7 +83,7 @@ capemz:
- user: 939
- group: 939
{% if grains['role'] != 'so-helix' %}
# Add ES Group
elasticsearchgroup:
@@ -315,27 +283,16 @@ auth_users_roles_inode:
so-elasticsearch:
docker_container.running:
- image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}
- image: {{ GLOBALS.manager }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ GLOBALS.so_version }}
- hostname: elasticsearch
- name: so-elasticsearch
- user: elasticsearch
- extra_hosts:
{% if ismanager %}
- {{ grains.host }}:{{ NODEIP }}
{% if salt['pillar.get']('nodestab', {}) %}
{% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
- {{ SN.split('_')|first }}:{{ SNDATA.ip }}
{% endfor %}
{% endif %}
{% else %}
- {{ grains.host }}:{{ NODEIP }}
- {{ MANAGER }}:{{ MANAGERIP }}
{% endif %}
- extra_hosts: {{ REDIS_NODES }}
- environment:
{% if TRUECLUSTER is sameas false or (TRUECLUSTER is sameas true and not salt['pillar.get']('nodestab', {})) %}
{% if REDIS_NODES | length == 1 %}
- discovery.type=single-node
{% endif %}
- ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true
- ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true
ulimits:
- memlock=-1:-1
- nofile=65536:65536
@@ -349,7 +306,7 @@ so-elasticsearch:
- /nsm/elasticsearch:/usr/share/elasticsearch/data:rw
- /opt/so/log/elasticsearch:/var/log/elasticsearch:rw
- /opt/so/conf/ca/cacerts:/usr/share/elasticsearch/jdk/lib/security/cacerts:ro
{% if ismanager %}
{% if GLOBALS.is_manager %}
- /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro
{% else %}
- /etc/ssl/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro
@@ -357,10 +314,8 @@ so-elasticsearch:
- /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro
- /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro
- /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro
{% if salt['pillar.get']('elasticsearch:auth:enabled', False) %}
- /opt/so/conf/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles:ro
- /opt/so/conf/elasticsearch/users:/usr/share/elasticsearch/config/users:ro
{% endif %}
{% if ESCONFIG.path.get('repo', False) %}
{% for repo in ESCONFIG.path.repo %}
- {{ repo }}:{{ repo }}:rw
@@ -378,15 +333,13 @@ so-elasticsearch:
- x509: /etc/pki/elasticsearch.crt
- x509: /etc/pki/elasticsearch.key
- file: elasticp12perms
{% if ismanager %}
{% if GLOBALS.is_manager %}
- x509: pki_public_ca_crt
{% else %}
- x509: trusttheca
{% endif %}
{% if salt['pillar.get']('elasticsearch:auth:enabled', False) %}
- cmd: auth_users_roles_inode
- cmd: auth_users_inode
{% endif %}
append_so-elasticsearch_so-status.conf:
file.append:
@@ -404,7 +357,7 @@ so-elasticsearch-templates:
so-elasticsearch-pipelines:
cmd.run:
- name: /usr/sbin/so-elasticsearch-pipelines {{ grains.host }}
- name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
- require:
- docker_container: so-elasticsearch
- file: so-elasticsearch-pipelines-script
@@ -418,7 +371,6 @@ so-elasticsearch-roles-load:
- docker_container: so-elasticsearch
- file: es_sync_scripts
{% endif %} {# if grains['role'] != 'so-helix' #}
{% else %}

104
salt/elasticsearch/soc_elasticsearch.yaml Normal file
View File

@@ -0,0 +1,104 @@
elasticsearch:
config:
cluster:
name:
description: The name of the Security Onion Elasticsearch cluster, for identification purposes.
readonly: True
global: True
routing:
allocation:
disk:
threshold_enabled:
description: Specifies whether the Elasticsearch node will monitor the available disk space for low disk space conditions and take action to protect the cluster.
watermark:
low:
description: The lower percentage of used disk space representing a healthy node.
high:
description: The higher percentage of used disk space representing an unhealthy node.
flood_stage:
description: The max percentage of used disk space that will cause the node to take protective actions, such as blocking incoming events.
script:
max_compilations_rate:
description: Max rate of script compilations permitted in the Elasticsearch cluster. Larger values will consume more resources.
global: True
indices:
query:
bool:
max_clause_count:
description: Max number of boolean clauses per query.
global: True
index_settings:
so-aws: &indexSettings
warm:
description: Age (in days) of this index before it will move to warm storage, if warm nodes are present. Once moved, events on this index can take longer to fetch.
global: True
close:
description: Age (in days) of this index before it will be closed. Once closed, events on this index cannot be retrieved without first re-opening the index.
global: True
delete:
description: Age (in days) of this index before it will be deleted. Once deleted, events are permanently unrecoverable.
global: True
index_sorting:
description: Sorts the index by event time, at the cost of additional processing resource consumption.
global: True
index_template:
template:
settings:
index:
mapping:
total_fields:
limit:
description: Max number of fields that can exist on a single index. Larger values will consume more resources.
global: True
refresh_interval:
description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
global: True
number_of_shards:
description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
global: True
number_of_replicas:
description: Number of replicas required for this index. Multiple replicas protects against data loss, while also increasing storage costs.
global: True
so-azure: *indexSettings
so-barracuda: *indexSettings
so-beats: *indexSettings
so-bluecoat: *indexSettings
so-cef: *indexSettings
so-checkpoint: *indexSettings
so-cisco: *indexSettings
so-cyberark: *indexSettings
so-cylance: *indexSettings
so-elasticsearch: *indexSettings
so-endgame: *indexSettings
so-f5: *indexSettings
so-firewall: *indexSettings
so-fortinet: *indexSettings
so-gcp: *indexSettings
so-google_workspace: *indexSettings
so-ids: *indexSettings
so-imperva: *indexSettings
so-import: *indexSettings
so-infoblox: *indexSettings
so-juniper: *indexSettings
so-kibana: *indexSettings
so-logstash: *indexSettings
so-microsoft: *indexSettings
so-misp: *indexSettings
so-netflow: *indexSettings
so-netscout: *indexSettings
so-o365: *indexSettings
so-okta: *indexSettings
so-osquery: *indexSettings
so-proofpoint: *indexSettings
so-radware: *indexSettings
so-redis: *indexSettings
so-snort: *indexSettings
so-snyk: *indexSettings
so-sonicwall: *indexSettings
so-sophos: *indexSettings
so-strelka: *indexSettings
so-syslog: *indexSettings
so-tomcat: *indexSettings
so-zeek: *indexSettings
so-zscaler: *indexSettings

20
salt/elasticsearch/tools/sbin/so-catrust
View File

@@ -1,19 +1,11 @@
#!/bin/bash
#
# Copyright 2014-2022 Security Onion Solutions, LLC
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set VERSION = salt['pillar.get']('global:soversion', '') %}
{%- set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
{%- set MANAGER = salt['grains.get']('master') %}

24
salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines
View File

@@ -1,19 +1,11 @@
#!/bin/bash
#
# Copyright 2014-2022 Security Onion Solutions, LLC
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
RETURN_CODE=0
ELASTICSEARCH_HOST=$1
@@ -30,7 +22,7 @@ if [ ! -f /opt/so/state/espipelines.txt ]; then
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 240 ]]; do
{{ ELASTICCURL }} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
@@ -50,7 +42,7 @@ if [ ! -f /opt/so/state/espipelines.txt ]; then
cd ${ELASTICSEARCH_INGEST_PIPELINES}
echo "Loading pipelines..."
for i in *; do echo $i; RESPONSE=$({{ ELASTICCURL }} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
for i in *; do echo $i; RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
echo
cd - >/dev/null

22
salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load
View File

@@ -1,18 +1,10 @@
#!/bin/bash
# Copyright 2014-2022 Security Onion Solutions, LLC
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set mainint = salt['pillar.get']('host:mainint') %}
{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %}
@@ -29,7 +21,7 @@ echo -n "Waiting for ElasticSearch..."
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 240 ]]; do
{{ ELASTICCURL }} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"

20
salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load Normal file → Executable file
View File

@@ -1,18 +1,10 @@
#!/bin/bash
# Copyright 2014-2022 Security Onion Solutions, LLC
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set mainint = salt['pillar.get']('host:mainint') %}
{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %}