From 7150564c70aaf903d22162870f0268a663fd4883 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 28 Jan 2020 18:58:26 -0500 Subject: [PATCH 1/7] heavynode --- pillar/top.sls | 9 +- salt/common/init.sls | 4 +- salt/common/nginx/nginx.conf.so-heavynode | 89 +++++++++++++ salt/elasticsearch/init.sls | 4 +- salt/filebeat/etc/filebeat.yml | 10 +- salt/filebeat/init.sls | 2 +- salt/firewall/init.sls | 40 +++++- .../master/templates/9999_output_redis.conf | 5 + .../search/templates/0900_input_redis.conf | 6 +- salt/logstash/etc/logstash.yml | 2 +- salt/logstash/init.sls | 2 +- salt/ssl/init.sls | 18 ++- salt/top.sls | 28 ++++ salt/wazuh/files/agent/ossec.conf | 2 +- salt/wazuh/files/agent/wazuh-register-agent | 2 +- setup/functions.sh | 10 +- setup/so-setup.sh | 126 ++++++++++++++++++ setup/whiptail.sh | 1 + 18 files changed, 339 insertions(+), 21 deletions(-) create mode 100644 salt/common/nginx/nginx.conf.so-heavynode diff --git a/pillar/top.sls b/pillar/top.sls index 99fe26556..852efe27c 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -2,7 +2,8 @@ base: '*': - patch.needs_restarting - 'G@role:so-mastersearch': + 'G@role:so-mastersearch or G@role:so-heavynode': + - match: compound - logstash.mastersearch 'G@role:so-sensor': @@ -32,6 +33,12 @@ base: - firewall.* - minions.{{ grains.id }} + 'G@role:so-heavynode': + - static + - firewall.* + - brologs + - minions.{{ grains.id }} + 'G@role:so-helix': - static - firewall.* diff --git a/salt/common/init.sls b/salt/common/init.sls index de7048c51..3098cdb50 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -1,6 +1,6 @@ {% set VERSION = salt['pillar.get']('static:soversion', '1.1.4') %} {% set MASTER = salt['grains.get']('master') %} -{%- set GRAFANA = salt['pillar.get']('master:grafana', '0') %} +{% set GRAFANA = salt['pillar.get']('master:grafana', '0') %} # Add socore Group socoregroup: group.present: @@ -343,7 +343,7 @@ dashboard-{{ SN }}: {% if salt['pillar.get']('nodestab', False) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} -dashboard-{{ SN }}: +dashboardsearch-{{ SN }}: file.managed: - name: /opt/so/conf/grafana/grafana_dashboards/search_nodes/{{ SN }}-Node.json - user: 939 diff --git a/salt/common/nginx/nginx.conf.so-heavynode b/salt/common/nginx/nginx.conf.so-heavynode new file mode 100644 index 000000000..39688f3df --- /dev/null +++ b/salt/common/nginx/nginx.conf.so-heavynode @@ -0,0 +1,89 @@ +# For more information on configuration, see: +# * Official English Documentation: http://nginx.org/en/docs/ +# * Official Russian Documentation: http://nginx.org/ru/docs/ + +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Load modular configuration files from the /etc/nginx/conf.d directory. + # See http://nginx.org/en/docs/ngx_core_module.html#include + # for more information. + include /etc/nginx/conf.d/*.conf; + + server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + root /usr/share/nginx/html; + + # Load configuration files for the default server block. + include /etc/nginx/default.d/*.conf; + + location / { + } + + error_page 404 /404.html; + location = /40x.html { + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + } + } + +# Settings for a TLS enabled server. +# +# server { +# listen 443 ssl http2 default_server; +# listen [::]:443 ssl http2 default_server; +# server_name _; +# root /usr/share/nginx/html; +# +# ssl_certificate "/etc/pki/nginx/server.crt"; +# ssl_certificate_key "/etc/pki/nginx/private/server.key"; +# ssl_session_cache shared:SSL:1m; +# ssl_session_timeout 10m; +# ssl_ciphers HIGH:!aNULL:!MD5; +# ssl_prefer_server_ciphers on; +# +# # Load configuration files for the default server block. +# include /etc/nginx/default.d/*.conf; +# +# location / { +# } +# +# error_page 404 /404.html; +# location = /40x.html { +# } +# +# error_page 500 502 503 504 /50x.html; +# location = /50x.html { +# } +# } + +} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 3d60f2925..79421003f 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -24,7 +24,7 @@ {% set esclustername = salt['pillar.get']('master:esclustername', '') %} {% set esheap = salt['pillar.get']('master:esheap', '') %} -{% elif grains['role'] == 'so-node' %} +{% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} {% set esclustername = salt['pillar.get']('node:esclustername', '') %} {% set esheap = salt['pillar.get']('node:esheap', '') %} @@ -135,4 +135,4 @@ so-elasticsearch-pipelines-file: so-elasticsearch-pipelines: cmd.run: - - name: /opt/so/conf/elasticsearch/so-elasticsearch-pipelines {{ esclustername }} \ No newline at end of file + - name: /opt/so/conf/elasticsearch/so-elasticsearch-pipelines {{ esclustername }} diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 4706e4c5a..45936c180 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -1,4 +1,10 @@ +{%- if grains.role == 'so-heavynode' %} +{%- set MASTER = grains.host %} +{%- else %} {%- set MASTER = grains['master'] %} +{%- endif %} + + {%- set HOSTNAME = salt['grains.get']('host', '') %} {%- set BROVER = salt['pillar.get']('static:broversion', 'COMMUNITY') %} {%- set WAZUHENABLED = salt['pillar.get']('static:wazuh_enabled', '1') %} @@ -67,12 +73,12 @@ filebeat.modules: # List of prospectors to fetch data. filebeat.prospectors: #------------------------------ Log prospector -------------------------------- -{%- if grains['role'] == 'so-sensor' or grains['role'] == "so-eval" or grains['role'] == "so-helix" %} +{%- if grains['role'] == 'so-sensor' or grains['role'] == "so-eval" or grains['role'] == "so-helix" or grains['role'] == "so-heavynode" %} {%- if BROVER != 'SURICATA' %} {%- for LOGNAME in salt['pillar.get']('brologs:enabled', '') %} - type: log paths: - - /nsm/bro/logs/current/{{ LOGNAME }}.log + - /nsm/zeek/logs/current/{{ LOGNAME }}.log fields: type: bro_{{ LOGNAME }} fields_under_root: true diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 1a59446b3..8f4985024 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -61,7 +61,7 @@ so-filebeat: - /opt/so/wazuh/logs/alerts/:/wazuh/alerts:ro - /opt/so/wazuh/logs/archives/:/wazuh/archives:ro - /opt/so/log/fleet/:/osquery/logs:ro -{%- if grains['role'] == 'so-master' %} +{%- if grains['role'] == 'so-master' or grains['role'] == 'so-heavynode' %} - /etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - /etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro {%- else %} diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index a016a9767..a26993cc0 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -1,7 +1,7 @@ # Firewall Magic for the grid {%- if grains['role'] in ['so-eval','so-master','so-helix','so-mastersearch'] %} {%- set ip = salt['pillar.get']('static:masterip', '') %} -{%- elif grains['role'] == 'so-node' %} +{%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} {%- set ip = salt['pillar.get']('node:mainip', '') %} {%- elif grains['role'] == 'so-sensor' %} {%- set ip = salt['pillar.get']('sensor:mainip', '') %} @@ -584,7 +584,7 @@ enable_standard_analyst_443_{{ip}}: {% endif %} # Rules if you are a Node -{% if grains['role'] == 'so-node' %} +{% if 'node' in grains['role'] %} #This should be more granular iptables_allow_docker: @@ -655,3 +655,39 @@ iptables_drop_all_the_things: - chain: LOGGING - jump: DROP - save: True + +{% if grains['role'] == 'so-heavynode' %} +# Allow Redis +enable_heavynode_redis_6379_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 6379 + - position: 1 + - save: True + +enable_forwardnode_beats_5044_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 5044 + - position: 1 + - save: True + +enable_forwardnode_beats_5644_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 5644 + - position: 1 + - save: True +{% endif %} diff --git a/salt/logstash/conf/pipelines/master/templates/9999_output_redis.conf b/salt/logstash/conf/pipelines/master/templates/9999_output_redis.conf index fa650b538..f176e0b94 100644 --- a/salt/logstash/conf/pipelines/master/templates/9999_output_redis.conf +++ b/salt/logstash/conf/pipelines/master/templates/9999_output_redis.conf @@ -1,10 +1,15 @@ {%- if salt['grains.get']('role') == 'so-master' %} {% set master = salt['pillar.get']('static:masterip', '') %} {%- set nodetype = 'master' %} +{% elif grains.role == 'so-heavynode' %} +{% set master = salt['pillar.get']('node:mainip', '') %} +{%- set nodetype = salt['pillar.get']('node:node_type', 'search') %} {%- else %} {%- set nodetype = salt['pillar.get']('node:node_type', 'storage') %} {% set master = salt['pillar.get']('static:masterip', '') %} {%- endif %} + + output { redis { host => '{{ master }}' diff --git a/salt/logstash/conf/pipelines/search/templates/0900_input_redis.conf b/salt/logstash/conf/pipelines/search/templates/0900_input_redis.conf index 660d2e741..ede940367 100644 --- a/salt/logstash/conf/pipelines/search/templates/0900_input_redis.conf +++ b/salt/logstash/conf/pipelines/search/templates/0900_input_redis.conf @@ -1,4 +1,8 @@ -{% set master = salt['pillar.get']('static:masterip', '') %} +{%- if grains.role == 'so-heavynode' %} +{%- set master = salt['pillar.get']('node:mainip', '') %} +{%- else %} +{%- set master = salt['pillar.get']('static:masterip', '') %} +{% endif -%} input { redis { host => '{{ master }}' diff --git a/salt/logstash/etc/logstash.yml b/salt/logstash/etc/logstash.yml index e5bb76b5b..6b07199d1 100644 --- a/salt/logstash/etc/logstash.yml +++ b/salt/logstash/etc/logstash.yml @@ -63,7 +63,7 @@ # # path.config: # /etc/logstash/conf.d is mapped to /usr/share/logstash/pipeline in the Docker image -{% if grains.role != 'so-mastersearch' %} +{% if grains.role != 'so-mastersearch' and grains.role != 'so-heavynode' %} path.config: /usr/share/logstash/pipeline.enabled/*.conf {% else %} #path.config: /usr/share/logstash/pipeline.enabled/*.conf diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 388019abf..8d84bf962 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -20,7 +20,7 @@ {% set lsheap = salt['pillar.get']('sensor:lsheap', '') %} {% set lsaccessip = salt['pillar.get']('sensor:lsaccessip', '') %} -{% elif grains['role'] == 'so-node' %} +{% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} {% set lsheap = salt['pillar.get']('node:lsheap', '') %} {% set nodetype = salt['pillar.get']('node:node_type', 'storage') %} diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index f3e815174..57579f6ca 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -2,7 +2,7 @@ {% set master_minion_id = master.split(".")[0] %} {%- set masterip = salt['pillar.get']('static:masterip', '') -%} -{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %} +{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-heavynode' %} {% set trusttheca_text = salt['mine.get'](grains.id, 'x509.get_pem_entries')[grains.id]['/etc/pki/ca.crt']|replace('\n', '') %} {% set ca_server = grains.id %} {% else %} @@ -41,7 +41,7 @@ m2cryptopkgs: bits: 4096 backup: True -{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' %} +{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-heavynode' %} # Request a cert and drop it where it needs to go to be distributed /etc/pki/filebeat.crt: @@ -49,7 +49,11 @@ m2cryptopkgs: - ca_server: {{ ca_server }} - signing_policy: filebeat - public_key: /etc/pki/filebeat.key - - CN: {{ master }} +{% if grains.role == 'so-heavynode' %} + - CN: {{grains.id}} +{% else %} + - CN: {{master}} +{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True @@ -129,7 +133,7 @@ fbcrtlink: backup: True {% endif %} -{% if grains['role'] == 'so-sensor' or grains['role'] == 'so-node' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' %} +{% if grains['role'] == 'so-sensor' or grains['role'] == 'so-node' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-heavynode' %} fbcertdir: file.directory: @@ -142,7 +146,11 @@ fbcertdir: - ca_server: {{ ca_server }} - signing_policy: filebeat - public_key: /opt/so/conf/filebeat/etc/pki/filebeat.key - - CN: {{ master }} +{% if grains.role == 'so-heavynode' %} + - CN: {{grains.id}} +{% else %} + - CN: {{master}} +{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True diff --git a/salt/top.sls b/salt/top.sls index 14556b67e..1608b93d6 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -233,3 +233,31 @@ base: {%- if DOMAINSTATS != 0 %} - domainstats {%- endif %} + + 'G@role:so-heavynode': + - ca + - ssl + - common + - firewall + - redis + - logstash + - elasticsearch + - curator + {%- if WAZUH != 0 %} + - wazuh + {%- endif %} + - filebeat + {%- if OSQUERY != 0 %} + - launcher + {%- endif %} + - pcap + - suricata + {%- if BROVER != 'SURICATA' %} + - zeek + {%- endif %} + - wazuh + - filebeat + {%- if OSQUERY != 0 %} + - launcher + {%- endif %} + - schedule diff --git a/salt/wazuh/files/agent/ossec.conf b/salt/wazuh/files/agent/ossec.conf index 8f54e6a08..c5a61e8ad 100644 --- a/salt/wazuh/files/agent/ossec.conf +++ b/salt/wazuh/files/agent/ossec.conf @@ -1,6 +1,6 @@ {%- if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-mastersearch' %} {%- set ip = salt['pillar.get']('static:masterip', '') %} -{%- elif grains['role'] == 'so-node' %} +{%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} {%- set ip = salt['pillar.get']('node:mainip', '') %} {%- elif grains['role'] == 'so-sensor' %} {%- set ip = salt['pillar.get']('sensor:mainip', '') %} diff --git a/salt/wazuh/files/agent/wazuh-register-agent b/salt/wazuh/files/agent/wazuh-register-agent index dbf4a4968..8a8c2ab7f 100755 --- a/salt/wazuh/files/agent/wazuh-register-agent +++ b/salt/wazuh/files/agent/wazuh-register-agent @@ -1,6 +1,6 @@ {%- if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-mastersearch' %} {%- set ip = salt['pillar.get']('static:masterip', '') %} -{%- elif grains['role'] == 'so-node' %} +{%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} {%- set ip = salt['pillar.get']('node:mainip', '') %} {%- elif grains['role'] == 'so-sensor' %} {%- set ip = salt['pillar.get']('sensor:mainip', '') %} diff --git a/setup/functions.sh b/setup/functions.sh index f23da15ba..95b992107 100644 --- a/setup/functions.sh +++ b/setup/functions.sh @@ -1281,6 +1281,14 @@ set_initial_firewall_policy() { ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM fi + if [ $INSTALLTYPE == 'HEAVYNODE' ]; then + ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP + ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP + ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh search_nodes $MAINIP + ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh sensorstab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM bond0 + ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM + fi + if [ $INSTALLTYPE == 'PARSINGNODE' ]; then echo "blah" fi @@ -1314,7 +1322,7 @@ set_management_interface() { set_node_type() { # Determine the node type based on whiplash choice - if [ $INSTALLTYPE == 'SEARCHNODE' ] || [ $INSTALLTYPE == 'EVALMODE' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then + if [ $INSTALLTYPE == 'SEARCHNODE' ] || [ $INSTALLTYPE == 'EVALMODE' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ] || [ $INSTALLTYPE == 'HEAVYNODE' ] ; then NODETYPE='search' fi if [ $INSTALLTYPE == 'PARSINGNODE' ]; then diff --git a/setup/so-setup.sh b/setup/so-setup.sh index a2501dfb9..8ee35a126 100644 --- a/setup/so-setup.sh +++ b/setup/so-setup.sh @@ -803,6 +803,132 @@ if (whiptail_you_sure) ; then fi + ######################## + ## Heavy Node ## + ######################## + + if [ $INSTALLTYPE == 'HEAVYNODE' ]; then + + filter_unused_nics + whiptail_bond_nics + whiptail_management_server + whiptail_master_updates + set_updates + whiptail_homenet_sensor + whiptail_sensor_config + # Calculate lbprocs so we can call it in the prompts + calculate_useable_cores + if [ $NSMSETUP == 'ADVANCED' ]; then + whiptail_bro_pins + whiptail_suricata_pins + whiptail_bond_nics_mtu + else + whiptail_basic_bro + whiptail_basic_suri + fi + + get_log_size_limit + CURCLOSEDAYS=30 + es_heapsize + ls_heapsize + whiptail_node_advanced + if [ $NODESETUP == 'NODEADVANCED' ]; then + whiptail_node_es_heap + whiptail_node_ls_heap + whiptail_node_ls_pipeline_worker + whiptail_node_ls_pipline_batchsize + whiptail_node_ls_input_threads + whiptail_node_ls_input_batch_count + whiptail_cur_close_days + whiptail_log_size_limit + else + NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE + NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE + LSPIPELINEWORKERS=$CPUCORES + LSPIPELINEBATCH=125 + LSINPUTTHREADS=1 + LSINPUTBATCHCOUNT=125 + fi + whiptail_make_changes + set_hostname + clear_master + mkdir -p /nsm + get_filesystem_root + get_filesystem_nsm + if [ $INSTALLMETHOD == iso ]; then + add_admin_user + disable_onion_user + fi + copy_ssh_key >> $SETUPLOG 2>&1 + { + sleep 0.5 + echo -e "XXX\n0\nSetting Initial Firewall Policy... \nXXX" + set_initial_firewall_policy >> $SETUPLOG 2>&1 + + echo -e "XXX\n3\nCreating Bond Interface... \nXXX" + create_sensor_bond >> $SETUPLOG 2>&1 + echo -e "XXX\n4\nGenerating Sensor Pillar... \nXXX" + sensor_pillar >> $SETUPLOG 2>&1 + echo "** Generating the patch pillar **" >> $SETUPLOG + patch_pillar >> $SETUPLOG 2>&1 + + + + echo -e "XXX\n5\nInstalling Salt Packages... \nXXX" + saltify >> $SETUPLOG 2>&1 + echo -e "XXX\n20\nInstalling Docker... \nXXX" + docker_install >> $SETUPLOG 2>&1 + echo -e "XXX\n30\nInitializing Minion... \nXXX" + configure_minion heavynode >> $SETUPLOG 2>&1 + set_node_type >> $SETUPLOG 2>&1 + node_pillar >> $SETUPLOG 2>&1 + echo -e "XXX\n24\nCopying Minion Pillars to Master... \nXXX" + copy_minion_tmp_files >> $SETUPLOG 2>&1 + echo -e "XXX\n35\nSending and Accepting Salt Key... \nXXX" + salt_firstcheckin >> $SETUPLOG 2>&1 + # Accept the Salt Key + accept_salt_key_remote >> $SETUPLOG 2>&1 + echo -e "XXX\n40\nApplying SSL Certificates... \nXXX" + salt-call state.apply ca >> $SETUPLOG 2>&1 + salt-call state.apply ssl >> $SETUPLOG 2>&1 + echo -e "XXX\n50\nConfiguring Firewall... \nXXX" + salt-call state.apply common >> $SETUPLOG 2>&1 + salt-call state.apply firewall >> $SETUPLOG 2>&1 + echo -e "XXX\n70\nInstalling Elastic Components... \nXXX" + salt-call state.apply logstash >> $SETUPLOG 2>&1 + salt-call state.apply elasticsearch >> $SETUPLOG 2>&1 + salt-call state.apply curator >> $SETUPLOG 2>&1 + salt-call state.apply filebeat >> $SETUPLOG 2>&1 + echo -e "XXX\n50\nInstalling PCAP... \nXXX" + salt-call state.apply pcap >> $SETUPLOG 2>&1 + echo -e "XXX\n60\nInstalling IDS components... \nXXX" + salt-call state.apply suricata >> $SETUPLOG 2>&1 + + checkin_at_boot >> $SETUPLOG 2>&1 + echo -e "XX\n97\nFinishing touches... \nXXX" + filter_unused_nics >> $SETUPLOG 2>&1 + network_setup >> $SETUPLOG 2>&1 + echo -e "XXX\n98\nVerifying Setup... \nXXX" + } |whiptail --title "Hybrid Hunter Install" --gauge "Please wait while installing" 6 60 0 + GOODSETUP=$(tail -10 $SETUPLOG | grep Failed | awk '{ print $2}') + if [[ $GOODSETUP == '0' ]]; then + whiptail_setup_complete + shutdown -r now + else + whiptail_setup_failed + shutdown -r now + fi + + fi + + + + + + + + + else echo "User not sure. Cancelling setup.">> $SETUPLOG 2>&1 whiptail_cancel diff --git a/setup/whiptail.sh b/setup/whiptail.sh index 171f180d2..3316d6e2c 100644 --- a/setup/whiptail.sh +++ b/setup/whiptail.sh @@ -254,6 +254,7 @@ whiptail_install_type() { "MASTERONLY" "Start a new grid" OFF \ "EVALMODE" "Evaluate all the things" OFF \ "MASTERSEARCH" "Master + Search Node" OFF \ + "HEAVYNODE" "Sensor + Search Node" OFF \ "HELIXSENSOR" "Connect this sensor to FireEye Helix" OFF \ "PARSINGNODE" "TODO Add a dedicated Parsing Node" OFF \ "HOTNODE" "TODO Add Hot Node (Search Node without Parsing)" OFF \ From b45241b42093697e069942db2d4f633d0ec3a675 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 29 Jan 2020 10:05:20 -0500 Subject: [PATCH 2/7] heavynode --- pillar/logstash/master.sls | 4 ++++ pillar/top.sls | 3 +++ salt/logstash/etc/logstash.yml | 2 +- salt/logstash/init.sls | 2 +- 4 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 pillar/logstash/master.sls diff --git a/pillar/logstash/master.sls b/pillar/logstash/master.sls new file mode 100644 index 000000000..3be98f6b9 --- /dev/null +++ b/pillar/logstash/master.sls @@ -0,0 +1,4 @@ +logstash: + pipelines: + master: + config: "/usr/share/logstash/pipelines/master/*.conf" diff --git a/pillar/top.sls b/pillar/top.sls index 852efe27c..8e8c22de3 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -20,6 +20,9 @@ base: - auth - minions.{{ grains.id }} + 'G@role:so-master': + - logstash.master + 'G@role:so-eval': - static - firewall.* diff --git a/salt/logstash/etc/logstash.yml b/salt/logstash/etc/logstash.yml index 6b07199d1..88f3c527e 100644 --- a/salt/logstash/etc/logstash.yml +++ b/salt/logstash/etc/logstash.yml @@ -63,7 +63,7 @@ # # path.config: # /etc/logstash/conf.d is mapped to /usr/share/logstash/pipeline in the Docker image -{% if grains.role != 'so-mastersearch' and grains.role != 'so-heavynode' %} +{% if grains.role != 'so-mastersearch' and grains.role != 'so-heavynode' and grains.role != 'so-master' %} path.config: /usr/share/logstash/pipeline.enabled/*.conf {% else %} #path.config: /usr/share/logstash/pipeline.enabled/*.conf diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 8d84bf962..4ac72af33 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -155,7 +155,7 @@ lscustsync: lsconfsync: file.managed: - name: /opt/so/conf/logstash/conf.enabled.txt -{% if grains.role == 'so-mastersearch' %} +{% if grains.role == 'so-mastersearch' or grains.role == 'so-heavynode' %} - source: salt://logstash/conf/conf.enabled.txt.so-master {% else %} - source: salt://logstash/conf/conf.enabled.txt.{{ nodetype }} From 306cc1127b295eb9086504eafd8dce32c18b52ca Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 28 Jan 2020 18:58:26 -0500 Subject: [PATCH 3/7] heavynode --- pillar/top.sls | 9 +- salt/common/init.sls | 4 +- salt/common/nginx/nginx.conf.so-heavynode | 89 +++++++++++++ salt/elasticsearch/init.sls | 2 +- salt/filebeat/etc/filebeat.yml | 8 +- salt/filebeat/init.sls | 2 +- salt/firewall/init.sls | 40 +++++- .../master/templates/9999_output_redis.conf | 5 + .../search/templates/0900_input_redis.conf | 6 +- salt/logstash/etc/logstash.yml | 2 +- salt/logstash/init.sls | 2 +- salt/ssl/init.sls | 18 ++- salt/top.sls | 28 ++++ salt/wazuh/files/agent/ossec.conf | 2 +- salt/wazuh/files/agent/wazuh-register-agent | 2 +- setup/functions.sh | 10 +- setup/so-setup.sh | 126 ++++++++++++++++++ setup/whiptail.sh | 1 + 18 files changed, 337 insertions(+), 19 deletions(-) create mode 100644 salt/common/nginx/nginx.conf.so-heavynode diff --git a/pillar/top.sls b/pillar/top.sls index 99fe26556..852efe27c 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -2,7 +2,8 @@ base: '*': - patch.needs_restarting - 'G@role:so-mastersearch': + 'G@role:so-mastersearch or G@role:so-heavynode': + - match: compound - logstash.mastersearch 'G@role:so-sensor': @@ -32,6 +33,12 @@ base: - firewall.* - minions.{{ grains.id }} + 'G@role:so-heavynode': + - static + - firewall.* + - brologs + - minions.{{ grains.id }} + 'G@role:so-helix': - static - firewall.* diff --git a/salt/common/init.sls b/salt/common/init.sls index 7ed59efa1..4ae78f57b 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -1,6 +1,6 @@ {% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} {% set MASTER = salt['grains.get']('master') %} -{%- set GRAFANA = salt['pillar.get']('master:grafana', '0') %} +{% set GRAFANA = salt['pillar.get']('master:grafana', '0') %} # Add socore Group socoregroup: group.present: @@ -343,7 +343,7 @@ dashboard-{{ SN }}: {% if salt['pillar.get']('nodestab', False) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} -dashboard-{{ SN }}: +dashboardsearch-{{ SN }}: file.managed: - name: /opt/so/conf/grafana/grafana_dashboards/search_nodes/{{ SN }}-Node.json - user: 939 diff --git a/salt/common/nginx/nginx.conf.so-heavynode b/salt/common/nginx/nginx.conf.so-heavynode new file mode 100644 index 000000000..39688f3df --- /dev/null +++ b/salt/common/nginx/nginx.conf.so-heavynode @@ -0,0 +1,89 @@ +# For more information on configuration, see: +# * Official English Documentation: http://nginx.org/en/docs/ +# * Official Russian Documentation: http://nginx.org/ru/docs/ + +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Load modular configuration files from the /etc/nginx/conf.d directory. + # See http://nginx.org/en/docs/ngx_core_module.html#include + # for more information. + include /etc/nginx/conf.d/*.conf; + + server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + root /usr/share/nginx/html; + + # Load configuration files for the default server block. + include /etc/nginx/default.d/*.conf; + + location / { + } + + error_page 404 /404.html; + location = /40x.html { + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + } + } + +# Settings for a TLS enabled server. +# +# server { +# listen 443 ssl http2 default_server; +# listen [::]:443 ssl http2 default_server; +# server_name _; +# root /usr/share/nginx/html; +# +# ssl_certificate "/etc/pki/nginx/server.crt"; +# ssl_certificate_key "/etc/pki/nginx/private/server.key"; +# ssl_session_cache shared:SSL:1m; +# ssl_session_timeout 10m; +# ssl_ciphers HIGH:!aNULL:!MD5; +# ssl_prefer_server_ciphers on; +# +# # Load configuration files for the default server block. +# include /etc/nginx/default.d/*.conf; +# +# location / { +# } +# +# error_page 404 /404.html; +# location = /40x.html { +# } +# +# error_page 500 502 503 504 /50x.html; +# location = /50x.html { +# } +# } + +} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index a2493091a..a97a2ae0f 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -31,7 +31,7 @@ {% set esclustername = salt['pillar.get']('master:esclustername', '') %} {% set esheap = salt['pillar.get']('master:esheap', '') %} -{% elif grains['role'] == 'so-node' %} +{% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} {% set esclustername = salt['pillar.get']('node:esclustername', '') %} {% set esheap = salt['pillar.get']('node:esheap', '') %} diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index a8308a6d9..45936c180 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -1,4 +1,10 @@ +{%- if grains.role == 'so-heavynode' %} +{%- set MASTER = grains.host %} +{%- else %} {%- set MASTER = grains['master'] %} +{%- endif %} + + {%- set HOSTNAME = salt['grains.get']('host', '') %} {%- set BROVER = salt['pillar.get']('static:broversion', 'COMMUNITY') %} {%- set WAZUHENABLED = salt['pillar.get']('static:wazuh_enabled', '1') %} @@ -67,7 +73,7 @@ filebeat.modules: # List of prospectors to fetch data. filebeat.prospectors: #------------------------------ Log prospector -------------------------------- -{%- if grains['role'] == 'so-sensor' or grains['role'] == "so-eval" or grains['role'] == "so-helix" %} +{%- if grains['role'] == 'so-sensor' or grains['role'] == "so-eval" or grains['role'] == "so-helix" or grains['role'] == "so-heavynode" %} {%- if BROVER != 'SURICATA' %} {%- for LOGNAME in salt['pillar.get']('brologs:enabled', '') %} - type: log diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 44cc7c65c..9d175943d 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -67,7 +67,7 @@ so-filebeat: - /opt/so/wazuh/logs/alerts/:/wazuh/alerts:ro - /opt/so/wazuh/logs/archives/:/wazuh/archives:ro - /opt/so/log/fleet/:/osquery/logs:ro -{%- if grains['role'] == 'so-master' %} +{%- if grains['role'] == 'so-master' or grains['role'] == 'so-heavynode' %} - /etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - /etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro {%- else %} diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index a016a9767..a26993cc0 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -1,7 +1,7 @@ # Firewall Magic for the grid {%- if grains['role'] in ['so-eval','so-master','so-helix','so-mastersearch'] %} {%- set ip = salt['pillar.get']('static:masterip', '') %} -{%- elif grains['role'] == 'so-node' %} +{%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} {%- set ip = salt['pillar.get']('node:mainip', '') %} {%- elif grains['role'] == 'so-sensor' %} {%- set ip = salt['pillar.get']('sensor:mainip', '') %} @@ -584,7 +584,7 @@ enable_standard_analyst_443_{{ip}}: {% endif %} # Rules if you are a Node -{% if grains['role'] == 'so-node' %} +{% if 'node' in grains['role'] %} #This should be more granular iptables_allow_docker: @@ -655,3 +655,39 @@ iptables_drop_all_the_things: - chain: LOGGING - jump: DROP - save: True + +{% if grains['role'] == 'so-heavynode' %} +# Allow Redis +enable_heavynode_redis_6379_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 6379 + - position: 1 + - save: True + +enable_forwardnode_beats_5044_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 5044 + - position: 1 + - save: True + +enable_forwardnode_beats_5644_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 5644 + - position: 1 + - save: True +{% endif %} diff --git a/salt/logstash/conf/pipelines/master/templates/9999_output_redis.conf b/salt/logstash/conf/pipelines/master/templates/9999_output_redis.conf index fa650b538..f176e0b94 100644 --- a/salt/logstash/conf/pipelines/master/templates/9999_output_redis.conf +++ b/salt/logstash/conf/pipelines/master/templates/9999_output_redis.conf @@ -1,10 +1,15 @@ {%- if salt['grains.get']('role') == 'so-master' %} {% set master = salt['pillar.get']('static:masterip', '') %} {%- set nodetype = 'master' %} +{% elif grains.role == 'so-heavynode' %} +{% set master = salt['pillar.get']('node:mainip', '') %} +{%- set nodetype = salt['pillar.get']('node:node_type', 'search') %} {%- else %} {%- set nodetype = salt['pillar.get']('node:node_type', 'storage') %} {% set master = salt['pillar.get']('static:masterip', '') %} {%- endif %} + + output { redis { host => '{{ master }}' diff --git a/salt/logstash/conf/pipelines/search/templates/0900_input_redis.conf b/salt/logstash/conf/pipelines/search/templates/0900_input_redis.conf index 660d2e741..ede940367 100644 --- a/salt/logstash/conf/pipelines/search/templates/0900_input_redis.conf +++ b/salt/logstash/conf/pipelines/search/templates/0900_input_redis.conf @@ -1,4 +1,8 @@ -{% set master = salt['pillar.get']('static:masterip', '') %} +{%- if grains.role == 'so-heavynode' %} +{%- set master = salt['pillar.get']('node:mainip', '') %} +{%- else %} +{%- set master = salt['pillar.get']('static:masterip', '') %} +{% endif -%} input { redis { host => '{{ master }}' diff --git a/salt/logstash/etc/logstash.yml b/salt/logstash/etc/logstash.yml index e5bb76b5b..6b07199d1 100644 --- a/salt/logstash/etc/logstash.yml +++ b/salt/logstash/etc/logstash.yml @@ -63,7 +63,7 @@ # # path.config: # /etc/logstash/conf.d is mapped to /usr/share/logstash/pipeline in the Docker image -{% if grains.role != 'so-mastersearch' %} +{% if grains.role != 'so-mastersearch' and grains.role != 'so-heavynode' %} path.config: /usr/share/logstash/pipeline.enabled/*.conf {% else %} #path.config: /usr/share/logstash/pipeline.enabled/*.conf diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 2d94c5354..dacbbbb00 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -27,7 +27,7 @@ {% set lsheap = salt['pillar.get']('sensor:lsheap', '') %} {% set lsaccessip = salt['pillar.get']('sensor:lsaccessip', '') %} -{% elif grains['role'] == 'so-node' %} +{% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} {% set lsheap = salt['pillar.get']('node:lsheap', '') %} {% set nodetype = salt['pillar.get']('node:node_type', 'storage') %} diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index f3e815174..57579f6ca 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -2,7 +2,7 @@ {% set master_minion_id = master.split(".")[0] %} {%- set masterip = salt['pillar.get']('static:masterip', '') -%} -{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %} +{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-heavynode' %} {% set trusttheca_text = salt['mine.get'](grains.id, 'x509.get_pem_entries')[grains.id]['/etc/pki/ca.crt']|replace('\n', '') %} {% set ca_server = grains.id %} {% else %} @@ -41,7 +41,7 @@ m2cryptopkgs: bits: 4096 backup: True -{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' %} +{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-heavynode' %} # Request a cert and drop it where it needs to go to be distributed /etc/pki/filebeat.crt: @@ -49,7 +49,11 @@ m2cryptopkgs: - ca_server: {{ ca_server }} - signing_policy: filebeat - public_key: /etc/pki/filebeat.key - - CN: {{ master }} +{% if grains.role == 'so-heavynode' %} + - CN: {{grains.id}} +{% else %} + - CN: {{master}} +{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True @@ -129,7 +133,7 @@ fbcrtlink: backup: True {% endif %} -{% if grains['role'] == 'so-sensor' or grains['role'] == 'so-node' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' %} +{% if grains['role'] == 'so-sensor' or grains['role'] == 'so-node' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-heavynode' %} fbcertdir: file.directory: @@ -142,7 +146,11 @@ fbcertdir: - ca_server: {{ ca_server }} - signing_policy: filebeat - public_key: /opt/so/conf/filebeat/etc/pki/filebeat.key - - CN: {{ master }} +{% if grains.role == 'so-heavynode' %} + - CN: {{grains.id}} +{% else %} + - CN: {{master}} +{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True diff --git a/salt/top.sls b/salt/top.sls index 14556b67e..1608b93d6 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -233,3 +233,31 @@ base: {%- if DOMAINSTATS != 0 %} - domainstats {%- endif %} + + 'G@role:so-heavynode': + - ca + - ssl + - common + - firewall + - redis + - logstash + - elasticsearch + - curator + {%- if WAZUH != 0 %} + - wazuh + {%- endif %} + - filebeat + {%- if OSQUERY != 0 %} + - launcher + {%- endif %} + - pcap + - suricata + {%- if BROVER != 'SURICATA' %} + - zeek + {%- endif %} + - wazuh + - filebeat + {%- if OSQUERY != 0 %} + - launcher + {%- endif %} + - schedule diff --git a/salt/wazuh/files/agent/ossec.conf b/salt/wazuh/files/agent/ossec.conf index 8f54e6a08..c5a61e8ad 100644 --- a/salt/wazuh/files/agent/ossec.conf +++ b/salt/wazuh/files/agent/ossec.conf @@ -1,6 +1,6 @@ {%- if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-mastersearch' %} {%- set ip = salt['pillar.get']('static:masterip', '') %} -{%- elif grains['role'] == 'so-node' %} +{%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} {%- set ip = salt['pillar.get']('node:mainip', '') %} {%- elif grains['role'] == 'so-sensor' %} {%- set ip = salt['pillar.get']('sensor:mainip', '') %} diff --git a/salt/wazuh/files/agent/wazuh-register-agent b/salt/wazuh/files/agent/wazuh-register-agent index dbf4a4968..8a8c2ab7f 100755 --- a/salt/wazuh/files/agent/wazuh-register-agent +++ b/salt/wazuh/files/agent/wazuh-register-agent @@ -1,6 +1,6 @@ {%- if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-mastersearch' %} {%- set ip = salt['pillar.get']('static:masterip', '') %} -{%- elif grains['role'] == 'so-node' %} +{%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} {%- set ip = salt['pillar.get']('node:mainip', '') %} {%- elif grains['role'] == 'so-sensor' %} {%- set ip = salt['pillar.get']('sensor:mainip', '') %} diff --git a/setup/functions.sh b/setup/functions.sh index 34de4a0e7..1fa8870ce 100644 --- a/setup/functions.sh +++ b/setup/functions.sh @@ -1283,6 +1283,14 @@ set_initial_firewall_policy() { ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM fi + if [ $INSTALLTYPE == 'HEAVYNODE' ]; then + ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP + ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP + ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh search_nodes $MAINIP + ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh sensorstab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM bond0 + ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM + fi + if [ $INSTALLTYPE == 'PARSINGNODE' ]; then echo "blah" fi @@ -1316,7 +1324,7 @@ set_management_interface() { set_node_type() { # Determine the node type based on whiplash choice - if [ $INSTALLTYPE == 'SEARCHNODE' ] || [ $INSTALLTYPE == 'EVALMODE' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then + if [ $INSTALLTYPE == 'SEARCHNODE' ] || [ $INSTALLTYPE == 'EVALMODE' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ] || [ $INSTALLTYPE == 'HEAVYNODE' ] ; then NODETYPE='search' fi if [ $INSTALLTYPE == 'PARSINGNODE' ]; then diff --git a/setup/so-setup.sh b/setup/so-setup.sh index a2501dfb9..8ee35a126 100644 --- a/setup/so-setup.sh +++ b/setup/so-setup.sh @@ -803,6 +803,132 @@ if (whiptail_you_sure) ; then fi + ######################## + ## Heavy Node ## + ######################## + + if [ $INSTALLTYPE == 'HEAVYNODE' ]; then + + filter_unused_nics + whiptail_bond_nics + whiptail_management_server + whiptail_master_updates + set_updates + whiptail_homenet_sensor + whiptail_sensor_config + # Calculate lbprocs so we can call it in the prompts + calculate_useable_cores + if [ $NSMSETUP == 'ADVANCED' ]; then + whiptail_bro_pins + whiptail_suricata_pins + whiptail_bond_nics_mtu + else + whiptail_basic_bro + whiptail_basic_suri + fi + + get_log_size_limit + CURCLOSEDAYS=30 + es_heapsize + ls_heapsize + whiptail_node_advanced + if [ $NODESETUP == 'NODEADVANCED' ]; then + whiptail_node_es_heap + whiptail_node_ls_heap + whiptail_node_ls_pipeline_worker + whiptail_node_ls_pipline_batchsize + whiptail_node_ls_input_threads + whiptail_node_ls_input_batch_count + whiptail_cur_close_days + whiptail_log_size_limit + else + NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE + NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE + LSPIPELINEWORKERS=$CPUCORES + LSPIPELINEBATCH=125 + LSINPUTTHREADS=1 + LSINPUTBATCHCOUNT=125 + fi + whiptail_make_changes + set_hostname + clear_master + mkdir -p /nsm + get_filesystem_root + get_filesystem_nsm + if [ $INSTALLMETHOD == iso ]; then + add_admin_user + disable_onion_user + fi + copy_ssh_key >> $SETUPLOG 2>&1 + { + sleep 0.5 + echo -e "XXX\n0\nSetting Initial Firewall Policy... \nXXX" + set_initial_firewall_policy >> $SETUPLOG 2>&1 + + echo -e "XXX\n3\nCreating Bond Interface... \nXXX" + create_sensor_bond >> $SETUPLOG 2>&1 + echo -e "XXX\n4\nGenerating Sensor Pillar... \nXXX" + sensor_pillar >> $SETUPLOG 2>&1 + echo "** Generating the patch pillar **" >> $SETUPLOG + patch_pillar >> $SETUPLOG 2>&1 + + + + echo -e "XXX\n5\nInstalling Salt Packages... \nXXX" + saltify >> $SETUPLOG 2>&1 + echo -e "XXX\n20\nInstalling Docker... \nXXX" + docker_install >> $SETUPLOG 2>&1 + echo -e "XXX\n30\nInitializing Minion... \nXXX" + configure_minion heavynode >> $SETUPLOG 2>&1 + set_node_type >> $SETUPLOG 2>&1 + node_pillar >> $SETUPLOG 2>&1 + echo -e "XXX\n24\nCopying Minion Pillars to Master... \nXXX" + copy_minion_tmp_files >> $SETUPLOG 2>&1 + echo -e "XXX\n35\nSending and Accepting Salt Key... \nXXX" + salt_firstcheckin >> $SETUPLOG 2>&1 + # Accept the Salt Key + accept_salt_key_remote >> $SETUPLOG 2>&1 + echo -e "XXX\n40\nApplying SSL Certificates... \nXXX" + salt-call state.apply ca >> $SETUPLOG 2>&1 + salt-call state.apply ssl >> $SETUPLOG 2>&1 + echo -e "XXX\n50\nConfiguring Firewall... \nXXX" + salt-call state.apply common >> $SETUPLOG 2>&1 + salt-call state.apply firewall >> $SETUPLOG 2>&1 + echo -e "XXX\n70\nInstalling Elastic Components... \nXXX" + salt-call state.apply logstash >> $SETUPLOG 2>&1 + salt-call state.apply elasticsearch >> $SETUPLOG 2>&1 + salt-call state.apply curator >> $SETUPLOG 2>&1 + salt-call state.apply filebeat >> $SETUPLOG 2>&1 + echo -e "XXX\n50\nInstalling PCAP... \nXXX" + salt-call state.apply pcap >> $SETUPLOG 2>&1 + echo -e "XXX\n60\nInstalling IDS components... \nXXX" + salt-call state.apply suricata >> $SETUPLOG 2>&1 + + checkin_at_boot >> $SETUPLOG 2>&1 + echo -e "XX\n97\nFinishing touches... \nXXX" + filter_unused_nics >> $SETUPLOG 2>&1 + network_setup >> $SETUPLOG 2>&1 + echo -e "XXX\n98\nVerifying Setup... \nXXX" + } |whiptail --title "Hybrid Hunter Install" --gauge "Please wait while installing" 6 60 0 + GOODSETUP=$(tail -10 $SETUPLOG | grep Failed | awk '{ print $2}') + if [[ $GOODSETUP == '0' ]]; then + whiptail_setup_complete + shutdown -r now + else + whiptail_setup_failed + shutdown -r now + fi + + fi + + + + + + + + + else echo "User not sure. Cancelling setup.">> $SETUPLOG 2>&1 whiptail_cancel diff --git a/setup/whiptail.sh b/setup/whiptail.sh index 171f180d2..3316d6e2c 100644 --- a/setup/whiptail.sh +++ b/setup/whiptail.sh @@ -254,6 +254,7 @@ whiptail_install_type() { "MASTERONLY" "Start a new grid" OFF \ "EVALMODE" "Evaluate all the things" OFF \ "MASTERSEARCH" "Master + Search Node" OFF \ + "HEAVYNODE" "Sensor + Search Node" OFF \ "HELIXSENSOR" "Connect this sensor to FireEye Helix" OFF \ "PARSINGNODE" "TODO Add a dedicated Parsing Node" OFF \ "HOTNODE" "TODO Add Hot Node (Search Node without Parsing)" OFF \ From bd5b597aed7716082b5dbde8597d71defadb2bcd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 29 Jan 2020 10:05:20 -0500 Subject: [PATCH 4/7] heavynode --- pillar/logstash/master.sls | 4 ++++ pillar/top.sls | 3 +++ salt/logstash/etc/logstash.yml | 2 +- salt/logstash/init.sls | 2 +- 4 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 pillar/logstash/master.sls diff --git a/pillar/logstash/master.sls b/pillar/logstash/master.sls new file mode 100644 index 000000000..3be98f6b9 --- /dev/null +++ b/pillar/logstash/master.sls @@ -0,0 +1,4 @@ +logstash: + pipelines: + master: + config: "/usr/share/logstash/pipelines/master/*.conf" diff --git a/pillar/top.sls b/pillar/top.sls index 852efe27c..8e8c22de3 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -20,6 +20,9 @@ base: - auth - minions.{{ grains.id }} + 'G@role:so-master': + - logstash.master + 'G@role:so-eval': - static - firewall.* diff --git a/salt/logstash/etc/logstash.yml b/salt/logstash/etc/logstash.yml index 6b07199d1..88f3c527e 100644 --- a/salt/logstash/etc/logstash.yml +++ b/salt/logstash/etc/logstash.yml @@ -63,7 +63,7 @@ # # path.config: # /etc/logstash/conf.d is mapped to /usr/share/logstash/pipeline in the Docker image -{% if grains.role != 'so-mastersearch' and grains.role != 'so-heavynode' %} +{% if grains.role != 'so-mastersearch' and grains.role != 'so-heavynode' and grains.role != 'so-master' %} path.config: /usr/share/logstash/pipeline.enabled/*.conf {% else %} #path.config: /usr/share/logstash/pipeline.enabled/*.conf diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index dacbbbb00..f92f047fa 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -162,7 +162,7 @@ lscustsync: lsconfsync: file.managed: - name: /opt/so/conf/logstash/conf.enabled.txt -{% if grains.role == 'so-mastersearch' %} +{% if grains.role == 'so-mastersearch' or grains.role == 'so-heavynode' %} - source: salt://logstash/conf/conf.enabled.txt.so-master {% else %} - source: salt://logstash/conf/conf.enabled.txt.{{ nodetype }} From 0d225020721989f58b126e13c57216bdb84eb8c0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 29 Jan 2020 16:45:04 -0500 Subject: [PATCH 5/7] changes to filebeat for heavynode --- salt/filebeat/init.sls | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 9d175943d..95736ca8b 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -1,5 +1,4 @@ # Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC - # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -12,16 +11,9 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} +{% set VERSION = salt['pillar.get']('static:soversion', '1.1.4') %} {% set MASTER = salt['grains.get']('master') %} {%- set MASTERIP = salt['pillar.get']('static:masterip', '') %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} -{% if FEATURES %} - {% set FEATURES = "-features" %} -{% else %} - {% set FEATURES = '' %} -{% endif %} - # Filebeat Setup filebeatetcdir: file.directory: @@ -29,21 +21,18 @@ filebeatetcdir: - user: 939 - group: 939 - makedirs: True - filebeatlogdir: file.directory: - name: /opt/so/log/filebeat - user: 939 - group: 939 - makedirs: True - filebeatpkidir: file.directory: - name: /opt/so/conf/filebeat/etc/pki - user: 939 - group: 939 - makedirs: True - # This needs to be owned by root filebeatconfsync: file.managed: @@ -52,10 +41,9 @@ filebeatconfsync: - user: 0 - group: 0 - template: jinja - so-filebeat: docker_container.running: - - image: {{ MASTER }}:5000/soshybridhunter/so-filebeat:{{ VERSION }}{{ FEATURES }} + - image: {{ MASTER }}:5000/soshybridhunter/so-filebeat:HH{{ VERSION }} - hostname: so-filebeat - user: root - extra_hosts: {{ MASTER }}:{{ MASTERIP }} @@ -67,13 +55,8 @@ so-filebeat: - /opt/so/wazuh/logs/alerts/:/wazuh/alerts:ro - /opt/so/wazuh/logs/archives/:/wazuh/archives:ro - /opt/so/log/fleet/:/osquery/logs:ro -{%- if grains['role'] == 'so-master' or grains['role'] == 'so-heavynode' %} - - /etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - - /etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro -{%- else %} - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro -{%- endif %} - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro - watch: - file: /opt/so/conf/filebeat/etc/filebeat.yml From 07df8bff7ea76316de9cdf830666798c508dc7f1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 29 Jan 2020 17:00:39 -0500 Subject: [PATCH 6/7] add filebeat features back --- salt/filebeat/init.sls | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 95736ca8b..8528ecc38 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -11,9 +11,15 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set VERSION = salt['pillar.get']('static:soversion', '1.1.4') %} +{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} {% set MASTER = salt['grains.get']('master') %} {%- set MASTERIP = salt['pillar.get']('static:masterip', '') %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} +{% if FEATURES %} + {% set FEATURES = "-features" %} +{% else %} + {% set FEATURES = '' %} +{% endif %} # Filebeat Setup filebeatetcdir: file.directory: @@ -43,7 +49,7 @@ filebeatconfsync: - template: jinja so-filebeat: docker_container.running: - - image: {{ MASTER }}:5000/soshybridhunter/so-filebeat:HH{{ VERSION }} + - image: {{ MASTER }}:5000/soshybridhunter/so-filebeat:{{ VERSION }}{{ FEATURES }} - hostname: so-filebeat - user: root - extra_hosts: {{ MASTER }}:{{ MASTERIP }} From 8b17d3ba6c0a2befea7a33c1d4f3ab56c374b259 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 29 Jan 2020 17:05:11 -0500 Subject: [PATCH 7/7] change ls heap for heavy node - issue 277 --- setup/so-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup.sh b/setup/so-setup.sh index 8ee35a126..b4051e32e 100644 --- a/setup/so-setup.sh +++ b/setup/so-setup.sh @@ -843,7 +843,7 @@ if (whiptail_you_sure) ; then whiptail_log_size_limit else NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE - NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE + NODE_LS_HEAP_SIZE=1000m LSPIPELINEWORKERS=$CPUCORES LSPIPELINEBATCH=125 LSINPUTTHREADS=1