Merge pull request #279 from Security-Onion-Solutions/features/heavynode

Features/heavynode
This commit is contained in:
weslambert
2020-01-29 17:07:00 -05:00
committed by GitHub
19 changed files with 349 additions and 35 deletions

View File

@@ -0,0 +1,4 @@
logstash:
pipelines:
master:
config: "/usr/share/logstash/pipelines/master/*.conf"

View File

@@ -2,7 +2,8 @@ base:
'*':
- patch.needs_restarting
'G@role:so-mastersearch':
'G@role:so-mastersearch or G@role:so-heavynode':
- match: compound
- logstash.mastersearch
'G@role:so-sensor':
@@ -19,6 +20,9 @@ base:
- auth
- minions.{{ grains.id }}
'G@role:so-master':
- logstash.master
'G@role:so-eval':
- static
- firewall.*
@@ -32,6 +36,12 @@ base:
- firewall.*
- minions.{{ grains.id }}
'G@role:so-heavynode':
- static
- firewall.*
- brologs
- minions.{{ grains.id }}
'G@role:so-helix':
- static
- firewall.*

View File

@@ -1,6 +1,6 @@
{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %}
{% set MASTER = salt['grains.get']('master') %}
{%- set GRAFANA = salt['pillar.get']('master:grafana', '0') %}
{% set GRAFANA = salt['pillar.get']('master:grafana', '0') %}
# Add socore Group
socoregroup:
group.present:
@@ -343,7 +343,7 @@ dashboard-{{ SN }}:
{% if salt['pillar.get']('nodestab', False) %}
{%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
dashboard-{{ SN }}:
dashboardsearch-{{ SN }}:
file.managed:
- name: /opt/so/conf/grafana/grafana_dashboards/search_nodes/{{ SN }}-Node.json
- user: 939

View File

@@ -0,0 +1,89 @@
# For more information on configuration, see:
# * Official English Documentation: http://nginx.org/en/docs/
# * Official Russian Documentation: http://nginx.org/ru/docs/
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
# Load dynamic modules. See /usr/share/nginx/README.dynamic.
include /usr/share/nginx/modules/*.conf;
events {
worker_connections 1024;
}
http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
include /etc/nginx/mime.types;
default_type application/octet-stream;
# Load modular configuration files from the /etc/nginx/conf.d directory.
# See http://nginx.org/en/docs/ngx_core_module.html#include
# for more information.
include /etc/nginx/conf.d/*.conf;
server {
listen 80 default_server;
listen [::]:80 default_server;
server_name _;
root /usr/share/nginx/html;
# Load configuration files for the default server block.
include /etc/nginx/default.d/*.conf;
location / {
}
error_page 404 /404.html;
location = /40x.html {
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
}
}
# Settings for a TLS enabled server.
#
# server {
# listen 443 ssl http2 default_server;
# listen [::]:443 ssl http2 default_server;
# server_name _;
# root /usr/share/nginx/html;
#
# ssl_certificate "/etc/pki/nginx/server.crt";
# ssl_certificate_key "/etc/pki/nginx/private/server.key";
# ssl_session_cache shared:SSL:1m;
# ssl_session_timeout 10m;
# ssl_ciphers HIGH:!aNULL:!MD5;
# ssl_prefer_server_ciphers on;
#
# # Load configuration files for the default server block.
# include /etc/nginx/default.d/*.conf;
#
# location / {
# }
#
# error_page 404 /404.html;
# location = /40x.html {
# }
#
# error_page 500 502 503 504 /50x.html;
# location = /50x.html {
# }
# }
}

View File

@@ -31,7 +31,7 @@
{% set esclustername = salt['pillar.get']('master:esclustername', '') %}
{% set esheap = salt['pillar.get']('master:esheap', '') %}
{% elif grains['role'] == 'so-node' %}
{% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %}
{% set esclustername = salt['pillar.get']('node:esclustername', '') %}
{% set esheap = salt['pillar.get']('node:esheap', '') %}

View File

@@ -1,4 +1,10 @@
{%- if grains.role == 'so-heavynode' %}
{%- set MASTER = grains.host %}
{%- else %}
{%- set MASTER = grains['master'] %}
{%- endif %}
{%- set HOSTNAME = salt['grains.get']('host', '') %}
{%- set BROVER = salt['pillar.get']('static:broversion', 'COMMUNITY') %}
{%- set WAZUHENABLED = salt['pillar.get']('static:wazuh_enabled', '1') %}
@@ -67,7 +73,7 @@ filebeat.modules:
# List of prospectors to fetch data.
filebeat.prospectors:
#------------------------------ Log prospector --------------------------------
{%- if grains['role'] == 'so-sensor' or grains['role'] == "so-eval" or grains['role'] == "so-helix" %}
{%- if grains['role'] == 'so-sensor' or grains['role'] == "so-eval" or grains['role'] == "so-helix" or grains['role'] == "so-heavynode" %}
{%- if BROVER != 'SURICATA' %}
{%- for LOGNAME in salt['pillar.get']('brologs:enabled', '') %}
- type: log

View File

@@ -1,5 +1,4 @@
# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
@@ -21,7 +20,6 @@
{% else %}
{% set FEATURES = '' %}
{% endif %}
# Filebeat Setup
filebeatetcdir:
file.directory:
@@ -29,21 +27,18 @@ filebeatetcdir:
- user: 939
- group: 939
- makedirs: True
filebeatlogdir:
file.directory:
- name: /opt/so/log/filebeat
- user: 939
- group: 939
- makedirs: True
filebeatpkidir:
file.directory:
- name: /opt/so/conf/filebeat/etc/pki
- user: 939
- group: 939
- makedirs: True
# This needs to be owned by root
filebeatconfsync:
file.managed:
@@ -52,7 +47,6 @@ filebeatconfsync:
- user: 0
- group: 0
- template: jinja
so-filebeat:
docker_container.running:
- image: {{ MASTER }}:5000/soshybridhunter/so-filebeat:{{ VERSION }}{{ FEATURES }}
@@ -67,13 +61,8 @@ so-filebeat:
- /opt/so/wazuh/logs/alerts/:/wazuh/alerts:ro
- /opt/so/wazuh/logs/archives/:/wazuh/archives:ro
- /opt/so/log/fleet/:/osquery/logs:ro
{%- if grains['role'] == 'so-master' %}
- /etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro
- /etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro
{%- else %}
- /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro
- /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro
{%- endif %}
- /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro
- watch:
- file: /opt/so/conf/filebeat/etc/filebeat.yml

View File

@@ -1,7 +1,7 @@
# Firewall Magic for the grid
{%- if grains['role'] in ['so-eval','so-master','so-helix','so-mastersearch'] %}
{%- set ip = salt['pillar.get']('static:masterip', '') %}
{%- elif grains['role'] == 'so-node' %}
{%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %}
{%- set ip = salt['pillar.get']('node:mainip', '') %}
{%- elif grains['role'] == 'so-sensor' %}
{%- set ip = salt['pillar.get']('sensor:mainip', '') %}
@@ -584,7 +584,7 @@ enable_standard_analyst_443_{{ip}}:
{% endif %}
# Rules if you are a Node
{% if grains['role'] == 'so-node' %}
{% if 'node' in grains['role'] %}
#This should be more granular
iptables_allow_docker:
@@ -655,3 +655,39 @@ iptables_drop_all_the_things:
- chain: LOGGING
- jump: DROP
- save: True
{% if grains['role'] == 'so-heavynode' %}
# Allow Redis
enable_heavynode_redis_6379_{{ip}}:
iptables.insert:
- table: filter
- chain: DOCKER-USER
- jump: ACCEPT
- proto: tcp
- source: {{ ip }}
- dport: 6379
- position: 1
- save: True
enable_forwardnode_beats_5044_{{ip}}:
iptables.insert:
- table: filter
- chain: DOCKER-USER
- jump: ACCEPT
- proto: tcp
- source: {{ ip }}
- dport: 5044
- position: 1
- save: True
enable_forwardnode_beats_5644_{{ip}}:
iptables.insert:
- table: filter
- chain: DOCKER-USER
- jump: ACCEPT
- proto: tcp
- source: {{ ip }}
- dport: 5644
- position: 1
- save: True
{% endif %}

View File

@@ -1,10 +1,15 @@
{%- if salt['grains.get']('role') == 'so-master' %}
{% set master = salt['pillar.get']('static:masterip', '') %}
{%- set nodetype = 'master' %}
{% elif grains.role == 'so-heavynode' %}
{% set master = salt['pillar.get']('node:mainip', '') %}
{%- set nodetype = salt['pillar.get']('node:node_type', 'search') %}
{%- else %}
{%- set nodetype = salt['pillar.get']('node:node_type', 'storage') %}
{% set master = salt['pillar.get']('static:masterip', '') %}
{%- endif %}
output {
redis {
host => '{{ master }}'

View File

@@ -1,4 +1,8 @@
{% set master = salt['pillar.get']('static:masterip', '') %}
{%- if grains.role == 'so-heavynode' %}
{%- set master = salt['pillar.get']('node:mainip', '') %}
{%- else %}
{%- set master = salt['pillar.get']('static:masterip', '') %}
{% endif -%}
input {
redis {
host => '{{ master }}'

View File

@@ -63,7 +63,7 @@
#
# path.config:
# /etc/logstash/conf.d is mapped to /usr/share/logstash/pipeline in the Docker image
{% if grains.role != 'so-mastersearch' %}
{% if grains.role != 'so-mastersearch' and grains.role != 'so-heavynode' and grains.role != 'so-master' %}
path.config: /usr/share/logstash/pipeline.enabled/*.conf
{% else %}
#path.config: /usr/share/logstash/pipeline.enabled/*.conf

View File

@@ -27,7 +27,7 @@
{% set lsheap = salt['pillar.get']('sensor:lsheap', '') %}
{% set lsaccessip = salt['pillar.get']('sensor:lsaccessip', '') %}
{% elif grains['role'] == 'so-node' %}
{% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %}
{% set lsheap = salt['pillar.get']('node:lsheap', '') %}
{% set nodetype = salt['pillar.get']('node:node_type', 'storage') %}
@@ -162,7 +162,7 @@ lscustsync:
lsconfsync:
file.managed:
- name: /opt/so/conf/logstash/conf.enabled.txt
{% if grains.role == 'so-mastersearch' %}
{% if grains.role == 'so-mastersearch' or grains.role == 'so-heavynode' %}
- source: salt://logstash/conf/conf.enabled.txt.so-master
{% else %}
- source: salt://logstash/conf/conf.enabled.txt.{{ nodetype }}

View File

@@ -2,7 +2,7 @@
{% set master_minion_id = master.split(".")[0] %}
{%- set masterip = salt['pillar.get']('static:masterip', '') -%}
{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %}
{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-heavynode' %}
{% set trusttheca_text = salt['mine.get'](grains.id, 'x509.get_pem_entries')[grains.id]['/etc/pki/ca.crt']|replace('\n', '') %}
{% set ca_server = grains.id %}
{% else %}
@@ -41,7 +41,7 @@ m2cryptopkgs:
bits: 4096
backup: True
{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' %}
{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-heavynode' %}
# Request a cert and drop it where it needs to go to be distributed
/etc/pki/filebeat.crt:
@@ -49,7 +49,11 @@ m2cryptopkgs:
- ca_server: {{ ca_server }}
- signing_policy: filebeat
- public_key: /etc/pki/filebeat.key
{% if grains.role == 'so-heavynode' %}
- CN: {{grains.id}}
{% else %}
- CN: {{master}}
{% endif %}
- days_remaining: 0
- days_valid: 820
- backup: True
@@ -129,7 +133,7 @@ fbcrtlink:
backup: True
{% endif %}
{% if grains['role'] == 'so-sensor' or grains['role'] == 'so-node' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' %}
{% if grains['role'] == 'so-sensor' or grains['role'] == 'so-node' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-heavynode' %}
fbcertdir:
file.directory:
@@ -142,7 +146,11 @@ fbcertdir:
- ca_server: {{ ca_server }}
- signing_policy: filebeat
- public_key: /opt/so/conf/filebeat/etc/pki/filebeat.key
{% if grains.role == 'so-heavynode' %}
- CN: {{grains.id}}
{% else %}
- CN: {{master}}
{% endif %}
- days_remaining: 0
- days_valid: 820
- backup: True

View File

@@ -233,3 +233,31 @@ base:
{%- if DOMAINSTATS != 0 %}
- domainstats
{%- endif %}
'G@role:so-heavynode':
- ca
- ssl
- common
- firewall
- redis
- logstash
- elasticsearch
- curator
{%- if WAZUH != 0 %}
- wazuh
{%- endif %}
- filebeat
{%- if OSQUERY != 0 %}
- launcher
{%- endif %}
- pcap
- suricata
{%- if BROVER != 'SURICATA' %}
- zeek
{%- endif %}
- wazuh
- filebeat
{%- if OSQUERY != 0 %}
- launcher
{%- endif %}
- schedule

View File

@@ -1,6 +1,6 @@
{%- if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-mastersearch' %}
{%- set ip = salt['pillar.get']('static:masterip', '') %}
{%- elif grains['role'] == 'so-node' %}
{%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %}
{%- set ip = salt['pillar.get']('node:mainip', '') %}
{%- elif grains['role'] == 'so-sensor' %}
{%- set ip = salt['pillar.get']('sensor:mainip', '') %}

View File

@@ -1,6 +1,6 @@
{%- if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-mastersearch' %}
{%- set ip = salt['pillar.get']('static:masterip', '') %}
{%- elif grains['role'] == 'so-node' %}
{%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %}
{%- set ip = salt['pillar.get']('node:mainip', '') %}
{%- elif grains['role'] == 'so-sensor' %}
{%- set ip = salt['pillar.get']('sensor:mainip', '') %}

View File

@@ -1283,6 +1283,14 @@ set_initial_firewall_policy() {
ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM
fi
if [ $INSTALLTYPE == 'HEAVYNODE' ]; then
ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP
ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP
ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh search_nodes $MAINIP
ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh sensorstab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM bond0
ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM
fi
if [ $INSTALLTYPE == 'PARSINGNODE' ]; then
echo "blah"
fi
@@ -1316,7 +1324,7 @@ set_management_interface() {
set_node_type() {
# Determine the node type based on whiplash choice
if [ $INSTALLTYPE == 'SEARCHNODE' ] || [ $INSTALLTYPE == 'EVALMODE' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then
if [ $INSTALLTYPE == 'SEARCHNODE' ] || [ $INSTALLTYPE == 'EVALMODE' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ] || [ $INSTALLTYPE == 'HEAVYNODE' ] ; then
NODETYPE='search'
fi
if [ $INSTALLTYPE == 'PARSINGNODE' ]; then

View File

@@ -803,6 +803,132 @@ if (whiptail_you_sure) ; then
fi
########################
## Heavy Node ##
########################
if [ $INSTALLTYPE == 'HEAVYNODE' ]; then
filter_unused_nics
whiptail_bond_nics
whiptail_management_server
whiptail_master_updates
set_updates
whiptail_homenet_sensor
whiptail_sensor_config
# Calculate lbprocs so we can call it in the prompts
calculate_useable_cores
if [ $NSMSETUP == 'ADVANCED' ]; then
whiptail_bro_pins
whiptail_suricata_pins
whiptail_bond_nics_mtu
else
whiptail_basic_bro
whiptail_basic_suri
fi
get_log_size_limit
CURCLOSEDAYS=30
es_heapsize
ls_heapsize
whiptail_node_advanced
if [ $NODESETUP == 'NODEADVANCED' ]; then
whiptail_node_es_heap
whiptail_node_ls_heap
whiptail_node_ls_pipeline_worker
whiptail_node_ls_pipline_batchsize
whiptail_node_ls_input_threads
whiptail_node_ls_input_batch_count
whiptail_cur_close_days
whiptail_log_size_limit
else
NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE
NODE_LS_HEAP_SIZE=1000m
LSPIPELINEWORKERS=$CPUCORES
LSPIPELINEBATCH=125
LSINPUTTHREADS=1
LSINPUTBATCHCOUNT=125
fi
whiptail_make_changes
set_hostname
clear_master
mkdir -p /nsm
get_filesystem_root
get_filesystem_nsm
if [ $INSTALLMETHOD == iso ]; then
add_admin_user
disable_onion_user
fi
copy_ssh_key >> $SETUPLOG 2>&1
{
sleep 0.5
echo -e "XXX\n0\nSetting Initial Firewall Policy... \nXXX"
set_initial_firewall_policy >> $SETUPLOG 2>&1
echo -e "XXX\n3\nCreating Bond Interface... \nXXX"
create_sensor_bond >> $SETUPLOG 2>&1
echo -e "XXX\n4\nGenerating Sensor Pillar... \nXXX"
sensor_pillar >> $SETUPLOG 2>&1
echo "** Generating the patch pillar **" >> $SETUPLOG
patch_pillar >> $SETUPLOG 2>&1
echo -e "XXX\n5\nInstalling Salt Packages... \nXXX"
saltify >> $SETUPLOG 2>&1
echo -e "XXX\n20\nInstalling Docker... \nXXX"
docker_install >> $SETUPLOG 2>&1
echo -e "XXX\n30\nInitializing Minion... \nXXX"
configure_minion heavynode >> $SETUPLOG 2>&1
set_node_type >> $SETUPLOG 2>&1
node_pillar >> $SETUPLOG 2>&1
echo -e "XXX\n24\nCopying Minion Pillars to Master... \nXXX"
copy_minion_tmp_files >> $SETUPLOG 2>&1
echo -e "XXX\n35\nSending and Accepting Salt Key... \nXXX"
salt_firstcheckin >> $SETUPLOG 2>&1
# Accept the Salt Key
accept_salt_key_remote >> $SETUPLOG 2>&1
echo -e "XXX\n40\nApplying SSL Certificates... \nXXX"
salt-call state.apply ca >> $SETUPLOG 2>&1
salt-call state.apply ssl >> $SETUPLOG 2>&1
echo -e "XXX\n50\nConfiguring Firewall... \nXXX"
salt-call state.apply common >> $SETUPLOG 2>&1
salt-call state.apply firewall >> $SETUPLOG 2>&1
echo -e "XXX\n70\nInstalling Elastic Components... \nXXX"
salt-call state.apply logstash >> $SETUPLOG 2>&1
salt-call state.apply elasticsearch >> $SETUPLOG 2>&1
salt-call state.apply curator >> $SETUPLOG 2>&1
salt-call state.apply filebeat >> $SETUPLOG 2>&1
echo -e "XXX\n50\nInstalling PCAP... \nXXX"
salt-call state.apply pcap >> $SETUPLOG 2>&1
echo -e "XXX\n60\nInstalling IDS components... \nXXX"
salt-call state.apply suricata >> $SETUPLOG 2>&1
checkin_at_boot >> $SETUPLOG 2>&1
echo -e "XX\n97\nFinishing touches... \nXXX"
filter_unused_nics >> $SETUPLOG 2>&1
network_setup >> $SETUPLOG 2>&1
echo -e "XXX\n98\nVerifying Setup... \nXXX"
} |whiptail --title "Hybrid Hunter Install" --gauge "Please wait while installing" 6 60 0
GOODSETUP=$(tail -10 $SETUPLOG | grep Failed | awk '{ print $2}')
if [[ $GOODSETUP == '0' ]]; then
whiptail_setup_complete
shutdown -r now
else
whiptail_setup_failed
shutdown -r now
fi
fi
else
echo "User not sure. Cancelling setup.">> $SETUPLOG 2>&1
whiptail_cancel

View File

@@ -254,6 +254,7 @@ whiptail_install_type() {
"MASTERONLY" "Start a new grid" OFF \
"EVALMODE" "Evaluate all the things" OFF \
"MASTERSEARCH" "Master + Search Node" OFF \
"HEAVYNODE" "Sensor + Search Node" OFF \
"HELIXSENSOR" "Connect this sensor to FireEye Helix" OFF \
"PARSINGNODE" "TODO Add a dedicated Parsing Node" OFF \
"HOTNODE" "TODO Add Hot Node (Search Node without Parsing)" OFF \