From d1641aa0d825454c2eaed29c1fdcfb04f0d08533 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 3 Aug 2020 15:49:18 -0400 Subject: [PATCH 01/36] chown /var/ossec dir to match the needful user/group ownership for ossec-agentd --- salt/wazuh/init.sls | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/wazuh/init.sls b/salt/wazuh/init.sls index 2695febd5..09c4e258b 100644 --- a/salt/wazuh/init.sls +++ b/salt/wazuh/init.sls @@ -46,6 +46,15 @@ wazuhpkgs: - hold: True - update_holds: True +wazuhvarossecdir: + file.directory: + - name: /var/ossec + - user: ossec + - group: ossec + - recurse: + - user + - group + # Add Wazuh agent conf wazuhagentconf: file.managed: From 7c1120e47d309bad331913113b829e036fd87969 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 3 Aug 2020 18:48:01 -0400 Subject: [PATCH 02/36] Fix grafana monitor interface. --- setup/so-functions | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index cb9c75437..ad4b4252f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1599,13 +1599,13 @@ set_initial_firewall_policy() { $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost search_node "$MAINIP" case "$install_type" in 'EVAL') - $default_salt_dir/pillar/data/addtotab.sh evaltab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" $INTERFACE True + $default_salt_dir/pillar/data/addtotab.sh evaltab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" "$INTERFACE" True ;; 'MANAGERSEARCH') $default_salt_dir/pillar/data/addtotab.sh managersearchtab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" ;; 'STANDALONE') - $default_salt_dir/pillar/data/addtotab.sh standalonetab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" $INTERFACE + $default_salt_dir/pillar/data/addtotab.sh standalonetab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" "$INTERFACE" ;; esac ;; @@ -1619,7 +1619,7 @@ set_initial_firewall_policy() { case "$install_type" in 'SENSOR') ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost sensor "$MAINIP" - ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" $INTERFACE + ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" "$INTERFACE" ;; 'SEARCHNODE') ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost search_node "$MAINIP" @@ -1628,7 +1628,7 @@ set_initial_firewall_policy() { 'HEAVYNODE') ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall includehost sensor "$MAINIP" ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost search_node "$MAINIP" - ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" $INTERFACE + ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" "$INTERFACE" ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh nodestab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" ;; 'FLEET') From 2290c28a07d838c22e2edfae71f9cfef28531ea5 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 4 Aug 2020 03:49:59 +0000 Subject: [PATCH 03/36] AWS defaults modifications --- setup/automation/aws_eval_defaults | 77 ++++++++++++++++++++++++ setup/automation/aws_standalone_defaults | 2 +- 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 setup/automation/aws_eval_defaults diff --git a/setup/automation/aws_eval_defaults b/setup/automation/aws_eval_defaults new file mode 100644 index 000000000..e038bf29d --- /dev/null +++ b/setup/automation/aws_eval_defaults @@ -0,0 +1,77 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +ALLOW_CIDR=0.0.0.0/0 +ALLOW_ROLE=a +BASICZEEK=7 +BASICSURI=7 +# BLOGS= +BNICS=ens6 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=eval-aws +install_type=EVAL +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=ens5 +# MSEARCH= +# MSRV= +# MTU= +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +# REDIRECTHOST= +REDIRECTINFO=HOSTNAME +RULESETUP=ETOPEN +# SHARDCOUNT= +SKIP_REBOOT=0 +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r diff --git a/setup/automation/aws_standalone_defaults b/setup/automation/aws_standalone_defaults index 3e27bd9e2..25d3da0e0 100644 --- a/setup/automation/aws_standalone_defaults +++ b/setup/automation/aws_standalone_defaults @@ -34,7 +34,7 @@ GRAFANA=1 # HELIXAPIKEY= HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 HNSENSOR=inherit -HOSTNAME=standalone +HOSTNAME=standalone-aws install_type=STANDALONE # LSINPUTBATCHCOUNT= # LSINPUTTHREADS= From 46f70c254ce013494247774dac7a09a5ad59a4c0 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 4 Aug 2020 14:11:50 +0000 Subject: [PATCH 04/36] Add AWS defaults file for manager --- setup/automation/aws_manager_defaults | 77 +++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 setup/automation/aws_manager_defaults diff --git a/setup/automation/aws_manager_defaults b/setup/automation/aws_manager_defaults new file mode 100644 index 000000000..2ca5c2a04 --- /dev/null +++ b/setup/automation/aws_manager_defaults @@ -0,0 +1,77 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +ALLOW_CIDR=0.0.0.0/0 +ALLOW_ROLE=a +BASICZEEK=7 +BASICSURI=7 +# BLOGS= +BNICS=ens6 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=manager-aws +install_type=MANAGER +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=ens5 +# MSEARCH= +# MSRV= +# MTU= +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +# REDIRECTHOST= +REDIRECTINFO=HOSTNAME +RULESETUP=ETOPEN +# SHARDCOUNT= +SKIP_REBOOT=0 +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r From 549bf7ba196ad8559d667d478a7ba12cb86d66bc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 10:17:43 -0400 Subject: [PATCH 05/36] Activate minio --- salt/minio/init.sls | 8 -------- salt/top.sls | 2 ++ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/salt/minio/init.sls b/salt/minio/init.sls index 2d5941301..fa9d2f2de 100644 --- a/salt/minio/init.sls +++ b/salt/minio/init.sls @@ -31,14 +31,6 @@ miniodatadir: - group: 939 - makedirs: True -#redisconfsync: -# file.recurse: -# - name: /opt/so/conf/redis/etc -# - source: salt://redis/etc -# - user: 939 -# - group: 939 -# - template: jinja - minio/minio: docker_image.present diff --git a/salt/top.sls b/salt/top.sls index 599f67dca..ff2fbfb0e 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -150,6 +150,7 @@ base: - wazuh {%- endif %} - logstash + - minio - kibana - elastalert - filebeat @@ -197,6 +198,7 @@ base: - wazuh {%- endif %} - logstash + - minio - kibana - pcap - suricata From 24ed92c9dc9b597523f041a17965944383b010a1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 15:54:03 -0400 Subject: [PATCH 06/36] minio and change to global --- pillar/docker/config.sls | 6 ++--- pillar/logstash/manager.sls | 2 +- pillar/logstash/search.sls | 2 +- pillar/top.sls | 18 +++++++-------- salt/common/maps/so-status.map.jinja | 8 +++---- salt/common/tools/sbin/so-elastic-clear | 2 +- salt/common/tools/sbin/so-features-enable | 6 ++--- salt/common/tools/sbin/so-import-pcap | 6 ++--- .../common/tools/sbin/so-kibana-config-export | 6 ++--- salt/common/tools/sbin/soup | 2 +- salt/curator/init.sls | 4 ++-- salt/deprecated-launcher/init.sls | 2 +- salt/domainstats/init.sls | 2 +- .../files/rules/so/suricata_thehive.yaml | 6 ++--- .../files/rules/so/wazuh_thehive.yaml | 6 ++--- salt/elastalert/init.sls | 4 ++-- salt/elasticsearch/init.sls | 4 ++-- salt/filebeat/etc/filebeat.yml | 8 +++---- salt/filebeat/init.sls | 6 ++--- salt/firewall/assigned_hostgroups.map.yaml | 8 +++++++ salt/firewall/portgroups.yaml | 3 +++ salt/fleet/event_gen-packages.sls | 10 ++++----- salt/fleet/event_update-custom-hostname.sls | 2 +- salt/fleet/init.sls | 6 ++--- salt/fleet/install_package.sls | 10 ++++----- salt/freqserver/init.sls | 2 +- salt/grafana/etc/datasources/influxdb.yaml | 2 +- salt/grafana/init.sls | 4 ++-- salt/idstools/init.sls | 4 ++-- salt/influxdb/init.sls | 4 ++-- salt/kibana/bin/so-kibana-config-load | 4 ++-- salt/kibana/init.sls | 4 ++-- salt/logstash/init.sls | 4 ++-- .../config/so/0899_input_minio.conf.jinja | 22 +++++++++++++++++++ .../config/so/0900_input_redis.conf.jinja | 2 +- .../config/so/9998_output_minio.conf.jinja | 17 ++++++++++++++ .../config/so/9999_output_redis.conf.jinja | 2 +- salt/manager/init.sls | 6 ++--- salt/minio/init.sls | 18 ++++++++++----- salt/mysql/init.sls | 8 +++---- salt/nginx/etc/nginx.conf.so-eval | 6 ++--- salt/nginx/etc/nginx.conf.so-manager | 6 ++--- salt/nginx/etc/nginx.conf.so-managersearch | 6 ++--- salt/nginx/etc/nginx.conf.so-standalone | 6 ++--- salt/nginx/files/navigator_config.json | 2 +- salt/nginx/init.sls | 8 +++---- salt/nodered/files/nodered_load_flows | 2 +- salt/nodered/files/so_flows.json | 6 ++--- salt/nodered/init.sls | 2 +- salt/pcap/files/sensoroni.json | 2 +- salt/pcap/init.sls | 4 ++-- salt/playbook/init.sls | 4 ++-- salt/reactor/fleet.sls | 2 +- salt/redis/init.sls | 4 ++-- salt/soc/files/soc/soc.json | 4 ++-- salt/soc/init.sls | 4 ++-- salt/soctopus/files/SOCtopus.conf | 4 ++-- .../files/templates/es-generic.template | 2 +- .../soctopus/files/templates/generic.template | 6 ++--- .../soctopus/files/templates/osquery.template | 6 ++--- salt/soctopus/init.sls | 6 ++--- salt/ssl/init.sls | 4 ++-- salt/strelka/files/backend/backend.yaml | 2 +- salt/strelka/files/filestream/filestream.yaml | 2 +- salt/strelka/files/frontend/frontend.yaml | 2 +- salt/strelka/files/manager/manager.yaml | 2 +- salt/strelka/init.sls | 6 ++--- salt/suricata/init.sls | 6 ++--- salt/suricata/suricata_config.map.jinja | 4 ++-- salt/tcpreplay/init.sls | 4 ++-- salt/telegraf/init.sls | 4 ++-- salt/thehive/etc/application.conf | 6 ++--- salt/thehive/etc/cortex-application.conf | 4 ++-- salt/thehive/init.sls | 4 ++-- salt/thehive/scripts/cortex_init | 20 ++++++++--------- salt/thehive/scripts/hive_init | 12 +++++----- salt/top.sls | 8 +++---- salt/wazuh/files/agent/ossec.conf | 2 +- salt/wazuh/files/agent/wazuh-register-agent | 2 +- salt/wazuh/files/wazuh-manager-whitelist | 4 ++-- salt/wazuh/init.sls | 4 ++-- salt/yum/etc/yum.conf.jinja | 2 +- salt/zeek/init.sls | 4 ++-- setup/so-functions | 21 +++++++++--------- setup/so-setup | 6 ++--- 85 files changed, 262 insertions(+), 207 deletions(-) create mode 100644 salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja create mode 100644 salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja diff --git a/pillar/docker/config.sls b/pillar/docker/config.sls index 4d70fd517..647151eef 100644 --- a/pillar/docker/config.sls +++ b/pillar/docker/config.sls @@ -1,11 +1,11 @@ -{%- set FLEETMANAGER = salt['pillar.get']('static:fleet_manager', False) -%} -{%- set FLEETNODE = salt['pillar.get']('static:fleet_node', False) -%} +{%- set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) -%} +{%- set FLEETNODE = salt['pillar.get']('global:fleet_node', False) -%} {% set WAZUH = salt['pillar.get']('manager:wazuh', '0') %} {% set THEHIVE = salt['pillar.get']('manager:thehive', '0') %} {% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %} {% set FREQSERVER = salt['pillar.get']('manager:freq', '0') %} {% set DOMAINSTATS = salt['pillar.get']('manager:domainstats', '0') %} -{% set ZEEKVER = salt['pillar.get']('static:zeekversion', 'COMMUNITY') %} +{% set ZEEKVER = salt['pillar.get']('global:zeekversion', 'COMMUNITY') %} {% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %} eval: diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls index 9c16d2625..861b8f665 100644 --- a/pillar/logstash/manager.sls +++ b/pillar/logstash/manager.sls @@ -4,4 +4,4 @@ logstash: config: - so/0009_input_beats.conf - so/0010_input_hhbeats.conf - - so/9999_output_redis.conf.jinja + - so/9998_output_minio.conf.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 486deb408..cad849153 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -2,7 +2,7 @@ logstash: pipelines: search: config: - - so/0900_input_redis.conf.jinja + - so/0899_input_minio.conf.jinja - so/9000_output_zeek.conf.jinja - so/9002_output_import.conf.jinja - so/9034_output_syslog.conf.jinja diff --git a/pillar/top.sls b/pillar/top.sls index 889f0b63f..c11b66eaa 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -14,14 +14,14 @@ base: - elasticsearch.search '*_sensor': - - static + - global - zeeklogs - healthcheck.sensor - minions.{{ grains.id }} '*_manager or *_managersearch': - match: compound - - static + - global - data.* - secrets - minions.{{ grains.id }} @@ -36,7 +36,7 @@ base: - secrets - healthcheck.eval - elasticsearch.eval - - static + - global - minions.{{ grains.id }} '*_standalone': @@ -48,20 +48,20 @@ base: - zeeklogs - secrets - healthcheck.standalone - - static + - global - minions.{{ grains.id }} '*_node': - - static + - global - minions.{{ grains.id }} '*_heavynode': - - static + - global - zeeklogs - minions.{{ grains.id }} '*_helix': - - static + - global - fireeye - zeeklogs - logstash @@ -69,13 +69,13 @@ base: - minions.{{ grains.id }} '*_fleet': - - static + - global - data.* - secrets - minions.{{ grains.id }} '*_searchnode': - - static + - global - logstash - logstash.search - elasticsearch.search diff --git a/salt/common/maps/so-status.map.jinja b/salt/common/maps/so-status.map.jinja index 93f5f3d13..21dd14ec9 100644 --- a/salt/common/maps/so-status.map.jinja +++ b/salt/common/maps/so-status.map.jinja @@ -20,7 +20,7 @@ {% if role in ['eval', 'managersearch', 'manager', 'standalone'] %} {{ append_containers('manager', 'grafana', 0) }} - {{ append_containers('static', 'fleet_manager', 0) }} + {{ append_containers('global', 'fleet_manager', 0) }} {{ append_containers('manager', 'wazuh', 0) }} {{ append_containers('manager', 'thehive', 0) }} {{ append_containers('manager', 'playbook', 0) }} @@ -29,11 +29,11 @@ {% endif %} {% if role in ['eval', 'heavynode', 'sensor', 'standalone'] %} - {{ append_containers('static', 'strelka', 0) }} + {{ append_containers('global', 'strelka', 0) }} {% endif %} {% if role in ['heavynode', 'standalone'] %} - {{ append_containers('static', 'zeekversion', 'SURICATA') }} + {{ append_containers('global', 'zeekversion', 'SURICATA') }} {% endif %} {% if role == 'searchnode' %} @@ -41,5 +41,5 @@ {% endif %} {% if role == 'sensor' %} - {{ append_containers('static', 'zeekversion', 'SURICATA') }} + {{ append_containers('global', 'zeekversion', 'SURICATA') }} {% endif %} \ No newline at end of file diff --git a/salt/common/tools/sbin/so-elastic-clear b/salt/common/tools/sbin/so-elastic-clear index 04c153f85..15b1041e1 100755 --- a/salt/common/tools/sbin/so-elastic-clear +++ b/salt/common/tools/sbin/so-elastic-clear @@ -14,7 +14,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{%- set MANAGERIP = salt['pillar.get']('static:managerip', '') -%} +{%- set MANAGERIP = salt['pillar.get']('global:managerip', '') -%} . /usr/sbin/so-common SKIP=0 diff --git a/salt/common/tools/sbin/so-features-enable b/salt/common/tools/sbin/so-features-enable index c94aebcba..070ecedc0 100755 --- a/salt/common/tools/sbin/so-features-enable +++ b/salt/common/tools/sbin/so-features-enable @@ -29,9 +29,9 @@ manager_check() { } manager_check -VERSION=$(grep soversion $local_salt_dir/pillar/static.sls | cut -d':' -f2|sed 's/ //g') -# Modify static.sls to enable Features -sed -i 's/features: False/features: True/' $local_salt_dir/pillar/static.sls +VERSION=$(grep soversion $local_salt_dir/pillar/global.sls | cut -d':' -f2|sed 's/ //g') +# Modify global.sls to enable Features +sed -i 's/features: False/features: True/' $local_salt_dir/pillar/global.sls SUFFIX="-features" TRUSTED_CONTAINERS=( \ "so-elasticsearch:$VERSION$SUFFIX" \ diff --git a/salt/common/tools/sbin/so-import-pcap b/salt/common/tools/sbin/so-import-pcap index aef6e98d8..6e2d98daa 100755 --- a/salt/common/tools/sbin/so-import-pcap +++ b/salt/common/tools/sbin/so-import-pcap @@ -16,9 +16,9 @@ # along with this program. If not, see . {% set MANAGER = salt['grains.get']('master') %} -{% set VERSION = salt['pillar.get']('static:soversion') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} -{%- set MANAGERIP = salt['pillar.get']('static:managerip') -%} +{% set VERSION = salt['pillar.get']('global:soversion') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} +{%- set MANAGERIP = salt['pillar.get']('global:managerip') -%} . /usr/sbin/so-common diff --git a/salt/common/tools/sbin/so-kibana-config-export b/salt/common/tools/sbin/so-kibana-config-export index 8ee3f59b5..6542c3f04 100755 --- a/salt/common/tools/sbin/so-kibana-config-export +++ b/salt/common/tools/sbin/so-kibana-config-export @@ -1,8 +1,8 @@ #!/bin/bash # -# {%- set FLEET_MANAGER = salt['pillar.get']('static:fleet_manager', False) -%} -# {%- set FLEET_NODE = salt['pillar.get']('static:fleet_node', False) -%} -# {%- set FLEET_IP = salt['pillar.get']('static:fleet_ip', '') %} +# {%- set FLEET_MANAGER = salt['pillar.get']('global:fleet_manager', False) -%} +# {%- set FLEET_NODE = salt['pillar.get']('global:fleet_node', False) -%} +# {%- set FLEET_IP = salt['pillar.get']('global:fleet_ip', '') %} # {%- set MANAGER = salt['pillar.get']('manager:url_base', '') %} # # Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index eb281baae..48d9314a3 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -159,7 +159,7 @@ update_version() { # Update the version to the latest echo "Updating the Security Onion version file." echo $NEWVERSION > /etc/soversion - sed -i "s/$INSTALLEDVERSION/$NEWVERSION/g" /opt/so/saltstack/local/pillar/static.sls + sed -i "s/$INSTALLEDVERSION/$NEWVERSION/g" /opt/so/saltstack/local/pillar/global.sls } upgrade_check() { diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 8873f401a..b98eaf6cb 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -1,5 +1,5 @@ -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% if grains['role'] in ['so-eval', 'so-node', 'so-managersearch', 'so-heavynode', 'so-standalone'] %} # Curator diff --git a/salt/deprecated-launcher/init.sls b/salt/deprecated-launcher/init.sls index 3ba9ad3a6..3805be5d7 100644 --- a/salt/deprecated-launcher/init.sls +++ b/salt/deprecated-launcher/init.sls @@ -1,4 +1,4 @@ -{%- set FLEETSETUP = salt['pillar.get']('static:fleetsetup', '0') -%} +{%- set FLEETSETUP = salt['pillar.get']('global:fleetsetup', '0') -%} {%- if FLEETSETUP != 0 %} launcherpkg: diff --git a/salt/domainstats/init.sls b/salt/domainstats/init.sls index 8d329c785..764435e5f 100644 --- a/salt/domainstats/init.sls +++ b/salt/domainstats/init.sls @@ -13,7 +13,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} # Create the group dstatsgroup: diff --git a/salt/elastalert/files/rules/so/suricata_thehive.yaml b/salt/elastalert/files/rules/so/suricata_thehive.yaml index fb6c6448d..0135edadd 100644 --- a/salt/elastalert/files/rules/so/suricata_thehive.yaml +++ b/salt/elastalert/files/rules/so/suricata_thehive.yaml @@ -1,6 +1,6 @@ -{% set es = salt['pillar.get']('static:managerip', '') %} -{% set hivehost = salt['pillar.get']('static:managerip', '') %} -{% set hivekey = salt['pillar.get']('static:hivekey', '') %} +{% set es = salt['pillar.get']('global:managerip', '') %} +{% set hivehost = salt['pillar.get']('global:managerip', '') %} +{% set hivekey = salt['pillar.get']('global:hivekey', '') %} {% set MANAGER = salt['pillar.get']('manager:url_base', '') %} # Elastalert rule to forward Suricata alerts from Security Onion to a specified TheHive instance. diff --git a/salt/elastalert/files/rules/so/wazuh_thehive.yaml b/salt/elastalert/files/rules/so/wazuh_thehive.yaml index c01bb5894..8aa085566 100644 --- a/salt/elastalert/files/rules/so/wazuh_thehive.yaml +++ b/salt/elastalert/files/rules/so/wazuh_thehive.yaml @@ -1,6 +1,6 @@ -{% set es = salt['pillar.get']('static:managerip', '') %} -{% set hivehost = salt['pillar.get']('static:managerip', '') %} -{% set hivekey = salt['pillar.get']('static:hivekey', '') %} +{% set es = salt['pillar.get']('global:managerip', '') %} +{% set hivehost = salt['pillar.get']('global:managerip', '') %} +{% set hivekey = salt['pillar.get']('global:hivekey', '') %} {% set MANAGER = salt['pillar.get']('manager:url_base', '') %} # Elastalert rule to forward high level Wazuh alerts from Security Onion to a specified TheHive instance. diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index 5703b8717..c6c3afb2f 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -12,8 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone'] %} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 909d30152..f3777481c 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -12,8 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 825ffaf64..2b8a4118f 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -6,11 +6,11 @@ {%- set HOSTNAME = salt['grains.get']('host', '') %} -{%- set ZEEKVER = salt['pillar.get']('static:zeekversion', 'COMMUNITY') %} -{%- set WAZUHENABLED = salt['pillar.get']('static:wazuh', '0') %} +{%- set ZEEKVER = salt['pillar.get']('global:zeekversion', 'COMMUNITY') %} +{%- set WAZUHENABLED = salt['pillar.get']('global:wazuh', '0') %} {%- set STRELKAENABLED = salt['pillar.get']('strelka:enabled', '0') %} -{%- set FLEETMANAGER = salt['pillar.get']('static:fleet_manager', False) -%} -{%- set FLEETNODE = salt['pillar.get']('static:fleet_node', False) -%} +{%- set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) -%} +{%- set FLEETNODE = salt['pillar.get']('global:fleet_node', False) -%} name: {{ HOSTNAME }} diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 0d1f521e3..a4fa36b14 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -11,10 +11,10 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} -{% set MANAGERIP = salt['pillar.get']('static:managerip', '') %} +{% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} {% if FEATURES %} {% set FEATURES = "-features" %} diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 2500c604a..7eb16a62a 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -15,6 +15,7 @@ role: - {{ portgroups.mysql }} - {{ portgroups.kibana }} - {{ portgroups.redis }} + - {{ portgroups.minio }} - {{ portgroups.influxdb }} - {{ portgroups.fleet_api }} - {{ portgroups.cortex }} @@ -38,6 +39,7 @@ role: search_node: portgroups: - {{ portgroups.redis }} + - {{ portgroups.minio }} - {{ portgroups.elasticsearch_node }} self: portgroups: @@ -99,6 +101,7 @@ role: - {{ portgroups.mysql }} - {{ portgroups.kibana }} - {{ portgroups.redis }} + - {{ portgroups.minio }} - {{ portgroups.influxdb }} - {{ portgroups.fleet_api }} - {{ portgroups.cortex }} @@ -122,6 +125,7 @@ role: search_node: portgroups: - {{ portgroups.redis }} + - {{ portgroups.minio }} - {{ portgroups.elasticsearch_node }} self: portgroups: @@ -180,6 +184,7 @@ role: - {{ portgroups.mysql }} - {{ portgroups.kibana }} - {{ portgroups.redis }} + - {{ portgroups.minio }} - {{ portgroups.influxdb }} - {{ portgroups.fleet_api }} - {{ portgroups.cortex }} @@ -203,6 +208,7 @@ role: search_node: portgroups: - {{ portgroups.redis }} + - {{ portgroups.minio }} - {{ portgroups.elasticsearch_node }} self: portgroups: @@ -261,6 +267,7 @@ role: - {{ portgroups.mysql }} - {{ portgroups.kibana }} - {{ portgroups.redis }} + - {{ portgroups.minio }} - {{ portgroups.influxdb }} - {{ portgroups.fleet_api }} - {{ portgroups.cortex }} @@ -284,6 +291,7 @@ role: search_node: portgroups: - {{ portgroups.redis }} + - {{ portgroups.minio }} - {{ portgroups.elasticsearch_node }} self: portgroups: diff --git a/salt/firewall/portgroups.yaml b/salt/firewall/portgroups.yaml index b8d86f253..5dee48755 100644 --- a/salt/firewall/portgroups.yaml +++ b/salt/firewall/portgroups.yaml @@ -45,6 +45,9 @@ firewall: kibana: tcp: - 5601 + minio: + tcp: + - 9595 mysql: tcp: - 3306 diff --git a/salt/fleet/event_gen-packages.sls b/salt/fleet/event_gen-packages.sls index 24b013704..bfcfd2a1d 100644 --- a/salt/fleet/event_gen-packages.sls +++ b/salt/fleet/event_gen-packages.sls @@ -1,10 +1,10 @@ {% set MANAGER = salt['grains.get']('master') %} {% set ENROLLSECRET = salt['pillar.get']('secrets:fleet_enroll-secret') %} -{% set CURRENTPACKAGEVERSION = salt['pillar.get']('static:fleet_packages-version') %} -{% set VERSION = salt['pillar.get']('static:soversion') %} -{% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('static:fleet_custom_hostname', None) %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} -{%- set FLEETNODE = salt['pillar.get']('static:fleet_node') -%} +{% set CURRENTPACKAGEVERSION = salt['pillar.get']('global:fleet_packages-version') %} +{% set VERSION = salt['pillar.get']('global:soversion') %} +{% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('global:fleet_custom_hostname', None) %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} +{%- set FLEETNODE = salt['pillar.get']('global:fleet_node') -%} {% if CUSTOM_FLEET_HOSTNAME != None and CUSTOM_FLEET_HOSTNAME != '' %} {% set HOSTNAME = CUSTOM_FLEET_HOSTNAME %} diff --git a/salt/fleet/event_update-custom-hostname.sls b/salt/fleet/event_update-custom-hostname.sls index 9278862ed..b404b2828 100644 --- a/salt/fleet/event_update-custom-hostname.sls +++ b/salt/fleet/event_update-custom-hostname.sls @@ -1,4 +1,4 @@ -{% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('static:fleet_custom_hostname', None) %} +{% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('global:fleet_custom_hostname', None) %} so/fleet: event.send: diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index 0b402a54b..b2a3bb516 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -1,8 +1,8 @@ {%- set MYSQLPASS = salt['pillar.get']('secrets:mysql', None) -%} {%- set FLEETPASS = salt['pillar.get']('secrets:fleet', None) -%} {%- set FLEETJWT = salt['pillar.get']('secrets:fleet_jwt', None) -%} -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set FLEETARCH = salt['grains.get']('role') %} @@ -10,7 +10,7 @@ {% set MAININT = salt['pillar.get']('host:mainint') %} {% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% else %} - {% set MAINIP = salt['pillar.get']('static:managerip') %} + {% set MAINIP = salt['pillar.get']('global:managerip') %} {% endif %} include: diff --git a/salt/fleet/install_package.sls b/salt/fleet/install_package.sls index d09de540c..9063464d8 100644 --- a/salt/fleet/install_package.sls +++ b/salt/fleet/install_package.sls @@ -1,8 +1,8 @@ -{%- set FLEETMANAGER = salt['pillar.get']('static:fleet_manager', False) -%} -{%- set FLEETNODE = salt['pillar.get']('static:fleet_node', False) -%} -{%- set FLEETHOSTNAME = salt['pillar.get']('static:fleet_hostname', False) -%} -{%- set FLEETIP = salt['pillar.get']('static:fleet_ip', False) -%} -{% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('static:fleet_custom_hostname', None) %} +{%- set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) -%} +{%- set FLEETNODE = salt['pillar.get']('global:fleet_node', False) -%} +{%- set FLEETHOSTNAME = salt['pillar.get']('global:fleet_hostname', False) -%} +{%- set FLEETIP = salt['pillar.get']('global:fleet_ip', False) -%} +{% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('global:fleet_custom_hostname', None) %} {% if CUSTOM_FLEET_HOSTNAME != (None and '') %} diff --git a/salt/freqserver/init.sls b/salt/freqserver/init.sls index 08661f3da..f48b66cff 100644 --- a/salt/freqserver/init.sls +++ b/salt/freqserver/init.sls @@ -13,7 +13,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} # Create the user fservergroup: diff --git a/salt/grafana/etc/datasources/influxdb.yaml b/salt/grafana/etc/datasources/influxdb.yaml index c70fd7137..a10bed981 100644 --- a/salt/grafana/etc/datasources/influxdb.yaml +++ b/salt/grafana/etc/datasources/influxdb.yaml @@ -1,4 +1,4 @@ -{%- set MANAGER = salt['pillar.get']('static:managerip', '') %} +{%- set MANAGER = salt['pillar.get']('global:managerip', '') %} apiVersion: 1 deleteDatasources: diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls index e63c9a9c4..eb446b2e0 100644 --- a/salt/grafana/init.sls +++ b/salt/grafana/init.sls @@ -1,7 +1,7 @@ {% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %} {% set MANAGER = salt['grains.get']('master') %} -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} diff --git a/salt/idstools/init.sls b/salt/idstools/init.sls index 3313fa901..93db83759 100644 --- a/salt/idstools/init.sls +++ b/salt/idstools/init.sls @@ -12,8 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} # IDSTools Setup idstoolsdir: diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 6d8ba4566..d35ab6cae 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -1,7 +1,7 @@ {% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %} {% set MANAGER = salt['grains.get']('master') %} -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index 451e848a1..2e5d38ade 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -1,6 +1,6 @@ #!/bin/bash -# {%- set FLEET_MANAGER = salt['pillar.get']('static:fleet_manager', False) -%} -# {%- set FLEET_NODE = salt['pillar.get']('static:fleet_node', False) -%} +# {%- set FLEET_MANAGER = salt['pillar.get']('global:fleet_manager', False) -%} +# {%- set FLEET_NODE = salt['pillar.get']('global:fleet_node', False) -%} # {%- set MANAGER = salt['pillar.get']('manager:url_base', '') %} KIBANA_VERSION="7.6.1" diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls index 9521c5bb1..a1dccd137 100644 --- a/salt/kibana/init.sls +++ b/salt/kibana/init.sls @@ -1,5 +1,5 @@ -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} {% if FEATURES %} diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 8a3b539a2..b63c1ce96 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -12,8 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} diff --git a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja new file mode 100644 index 000000000..1f6bf03b4 --- /dev/null +++ b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja @@ -0,0 +1,22 @@ +{%- if grains.role == 'so-heavynode' %} +{%- set MANAGER = salt['pillar.get']('elasticsearch:mainip', '') %} +{%- else %} +{%- set MANAGER = salt['pillar.get']('global:managerip', '') %} +{% endif -%} +{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} +{%- set access_key = salt['pillar.get']('global:access_key', '') %} +{%- set access_secret = salt['pillar.get']('global:access_secret', '') %} +input { + s3 { + access_key_id => "{{ access_key }}" + secret_access_key => "{{ access_secret }}" + endpoint => "http://{{ MANAGER }}:9595" + bucket => "logstash" + delete => true + interval => 10 + codec => json + additional_settings => { + "force_path_style" => true + } + } +} diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja index 2ce204875..6e736f22f 100644 --- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja @@ -1,7 +1,7 @@ {%- if grains.role == 'so-heavynode' %} {%- set MANAGER = salt['pillar.get']('elasticsearch:mainip', '') %} {%- else %} -{%- set MANAGER = salt['pillar.get']('static:managerip', '') %} +{%- set MANAGER = salt['pillar.get']('global:managerip', '') %} {% endif -%} {%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} diff --git a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja new file mode 100644 index 000000000..a085ee587 --- /dev/null +++ b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja @@ -0,0 +1,17 @@ +{%- set MANAGER = salt['pillar.get']('global:managerip', '') -%} +{%- set access_key = salt['pillar.get']('global:access_key', '') %} +{%- set access_secret = salt['pillar.get']('global:access_secret', '') %} +output { + s3 { + access_key_id => "{{ access_key }}" + secret_access_key => "{{ access_secret}}" + endpoint => "http://{{ MANAGER }}:9595" + bucket => "logstash" + size_file => 2048 + time_file => 1 + codec => json + additional_settings => { + "force_path_style" => true + } + } +} diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index 71ec9f639..239ca8cb6 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -1,4 +1,4 @@ -{% set MANAGER = salt['pillar.get']('static:managerip', '') %} +{% set MANAGER = salt['pillar.get']('global:managerip', '') %} {% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} output { redis { diff --git a/salt/manager/init.sls b/salt/manager/init.sls index aef705724..3b4852542 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -12,10 +12,10 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} -{% set managerproxy = salt['pillar.get']('static:managerupdate', '0') %} +{% set managerproxy = salt['pillar.get']('global:managerupdate', '0') %} socore_own_saltstack: file.directory: diff --git a/salt/minio/init.sls b/salt/minio/init.sls index fa9d2f2de..438face99 100644 --- a/salt/minio/init.sls +++ b/salt/minio/init.sls @@ -13,8 +13,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set access_key = salt['pillar.get']('manager:access_key', '') %} -{% set access_secret = salt['pillar.get']('manager:access_secret', '') %} +{% set access_key = salt['pillar.get']('minio:access_key', '') %} +{% set access_secret = salt['pillar.get']('minio:access_secret', '') %} # Minio Setup minioconfdir: @@ -26,7 +26,14 @@ minioconfdir: miniodatadir: file.directory: - - name: /nsm/minio/data + - name: /nsm/minio/data/ + - user: 939 + - group: 939 + - makedirs: True + +logstashbucket: + file.directory: + - name: /nsm/minio/data/logstash - user: 939 - group: 939 - makedirs: True @@ -40,12 +47,11 @@ minio: - hostname: so-minio - user: socore - port_bindings: - - 0.0.0.0:9000:9000 + - 0.0.0.0:9595:9595 - environment: - MINIO_ACCESS_KEY: {{ access_key }} - MINIO_SECRET_KEY: {{ access_secret }} - binds: - /nsm/minio/data:/data:rw - /opt/so/conf/minio/etc:/root/.minio:rw - - entrypoint: "/usr/bin/docker-entrypoint.sh server /data" - - network_mode: so-elastic-net + - entrypoint: "/usr/bin/docker-entrypoint.sh server --address :9595 /data" diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index 78240fe2f..c9c6fde41 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -1,7 +1,7 @@ {%- set MYSQLPASS = salt['pillar.get']('secrets:mysql', None) %} -{%- set MANAGERIP = salt['pillar.get']('static:managerip', '') %} -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set MAINIP = salt['pillar.get']('elasticsearch:mainip') %} {% set FLEETARCH = salt['grains.get']('role') %} @@ -10,7 +10,7 @@ {% set MAININT = salt['pillar.get']('host:mainint') %} {% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% else %} - {% set MAINIP = salt['pillar.get']('static:managerip') %} + {% set MAINIP = salt['pillar.get']('global:managerip') %} {% endif %} # MySQL Setup diff --git a/salt/nginx/etc/nginx.conf.so-eval b/salt/nginx/etc/nginx.conf.so-eval index 2998a5bf2..9c919c764 100644 --- a/salt/nginx/etc/nginx.conf.so-eval +++ b/salt/nginx/etc/nginx.conf.so-eval @@ -1,7 +1,7 @@ {%- set managerip = salt['pillar.get']('manager:mainip', '') %} -{%- set FLEET_MANAGER = salt['pillar.get']('static:fleet_manager') %} -{%- set FLEET_NODE = salt['pillar.get']('static:fleet_node') %} -{%- set FLEET_IP = salt['pillar.get']('static:fleet_ip', None) %} +{%- set FLEET_MANAGER = salt['pillar.get']('global:fleet_manager') %} +{%- set FLEET_NODE = salt['pillar.get']('global:fleet_node') %} +{%- set FLEET_IP = salt['pillar.get']('global:fleet_ip', None) %} # For more information on configuration, see: # * Official English Documentation: http://nginx.org/en/docs/ # * Official Russian Documentation: http://nginx.org/ru/docs/ diff --git a/salt/nginx/etc/nginx.conf.so-manager b/salt/nginx/etc/nginx.conf.so-manager index bdb342cac..cf7545942 100644 --- a/salt/nginx/etc/nginx.conf.so-manager +++ b/salt/nginx/etc/nginx.conf.so-manager @@ -1,7 +1,7 @@ {%- set managerip = salt['pillar.get']('manager:mainip', '') %} -{%- set FLEET_MANAGER = salt['pillar.get']('static:fleet_manager') %} -{%- set FLEET_NODE = salt['pillar.get']('static:fleet_node') %} -{%- set FLEET_IP = salt['pillar.get']('static:fleet_ip', None) %} +{%- set FLEET_MANAGER = salt['pillar.get']('global:fleet_manager') %} +{%- set FLEET_NODE = salt['pillar.get']('global:fleet_node') %} +{%- set FLEET_IP = salt['pillar.get']('global:fleet_ip', None) %} # For more information on configuration, see: # * Official English Documentation: http://nginx.org/en/docs/ # * Official Russian Documentation: http://nginx.org/ru/docs/ diff --git a/salt/nginx/etc/nginx.conf.so-managersearch b/salt/nginx/etc/nginx.conf.so-managersearch index cb7576923..4b9daba4e 100644 --- a/salt/nginx/etc/nginx.conf.so-managersearch +++ b/salt/nginx/etc/nginx.conf.so-managersearch @@ -1,7 +1,7 @@ {%- set managerip = salt['pillar.get']('manager:mainip', '') %} -{%- set FLEET_MANAGER = salt['pillar.get']('static:fleet_manager') %} -{%- set FLEET_NODE = salt['pillar.get']('static:fleet_node') %} -{%- set FLEET_IP = salt['pillar.get']('static:fleet_ip', None) %} +{%- set FLEET_MANAGER = salt['pillar.get']('global:fleet_manager') %} +{%- set FLEET_NODE = salt['pillar.get']('global:fleet_node') %} +{%- set FLEET_IP = salt['pillar.get']('global:fleet_ip', None) %} # For more information on configuration, see: # * Official English Documentation: http://nginx.org/en/docs/ # * Official Russian Documentation: http://nginx.org/ru/docs/ diff --git a/salt/nginx/etc/nginx.conf.so-standalone b/salt/nginx/etc/nginx.conf.so-standalone index bdb342cac..cf7545942 100644 --- a/salt/nginx/etc/nginx.conf.so-standalone +++ b/salt/nginx/etc/nginx.conf.so-standalone @@ -1,7 +1,7 @@ {%- set managerip = salt['pillar.get']('manager:mainip', '') %} -{%- set FLEET_MANAGER = salt['pillar.get']('static:fleet_manager') %} -{%- set FLEET_NODE = salt['pillar.get']('static:fleet_node') %} -{%- set FLEET_IP = salt['pillar.get']('static:fleet_ip', None) %} +{%- set FLEET_MANAGER = salt['pillar.get']('global:fleet_manager') %} +{%- set FLEET_NODE = salt['pillar.get']('global:fleet_node') %} +{%- set FLEET_IP = salt['pillar.get']('global:fleet_ip', None) %} # For more information on configuration, see: # * Official English Documentation: http://nginx.org/en/docs/ # * Official Russian Documentation: http://nginx.org/ru/docs/ diff --git a/salt/nginx/files/navigator_config.json b/salt/nginx/files/navigator_config.json index bd40e09ef..d54f13265 100644 --- a/salt/nginx/files/navigator_config.json +++ b/salt/nginx/files/navigator_config.json @@ -1,4 +1,4 @@ -{%- set ip = salt['pillar.get']('static:managerip', '') %} +{%- set ip = salt['pillar.get']('global:managerip', '') %} { "enterprise_attack_url": "https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json", diff --git a/salt/nginx/init.sls b/salt/nginx/init.sls index 53bb13eec..2e67a6b2c 100644 --- a/salt/nginx/init.sls +++ b/salt/nginx/init.sls @@ -1,8 +1,8 @@ -{% set FLEETMANAGER = salt['pillar.get']('static:fleet_manager', False) %} -{% set FLEETNODE = salt['pillar.get']('static:fleet_node', False) %} +{% set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) %} +{% set FLEETNODE = salt['pillar.get']('global:fleet_node', False) %} {% set MANAGER = salt['grains.get']('master') %} -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} # Drop the correct nginx config based on role nginxconfdir: diff --git a/salt/nodered/files/nodered_load_flows b/salt/nodered/files/nodered_load_flows index 985c1c49a..78bab818a 100644 --- a/salt/nodered/files/nodered_load_flows +++ b/salt/nodered/files/nodered_load_flows @@ -1,4 +1,4 @@ -{%- set ip = salt['pillar.get']('static:managerip', '') -%} +{%- set ip = salt['pillar.get']('global:managerip', '') -%} #!/bin/bash default_salt_dir=/opt/so/saltstack/default diff --git a/salt/nodered/files/so_flows.json b/salt/nodered/files/so_flows.json index ad780ceb9..a8a6e2c69 100644 --- a/salt/nodered/files/so_flows.json +++ b/salt/nodered/files/so_flows.json @@ -1,4 +1,4 @@ -{%- set MANAGERIP = salt['pillar.get']('static:managerip', '') -%} -{%- set HIVEKEY = salt['pillar.get']('static:hivekey', '') -%} -{%- set CORTEXKEY = salt['pillar.get']('static:cortexkey', '') -%} +{%- set MANAGERIP = salt['pillar.get']('global:managerip', '') -%} +{%- set HIVEKEY = salt['pillar.get']('global:hivekey', '') -%} +{%- set CORTEXKEY = salt['pillar.get']('global:cortexkey', '') -%} [{"id":"dca608c3.7d8af8","type":"tab","label":"TheHive - Webhook Events","disabled":false,"info":""},{"id":"4db74fa6.2556d","type":"tls-config","z":"","name":"","cert":"","key":"","ca":"","certname":"","keyname":"","caname":"","servername":"","verifyservercert":false},{"id":"aa6cf50d.a02fc8","type":"http in","z":"dca608c3.7d8af8","name":"TheHive Listener","url":"/thehive","method":"post","upload":false,"swaggerDoc":"","x":120,"y":780,"wires":[["2b92aebb.853dc2","2fce29bb.1b1376","82ad0f08.7a53f"]]},{"id":"2b92aebb.853dc2","type":"debug","z":"dca608c3.7d8af8","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","x":470,"y":940,"wires":[]},{"id":"a4ecb84a.805958","type":"switch","z":"dca608c3.7d8af8","name":"Operation","property":"payload.operation","propertyType":"msg","rules":[{"t":"eq","v":"Creation","vt":"str"},{"t":"eq","v":"Update","vt":"str"},{"t":"eq","v":"Delete","vt":"str"}],"checkall":"false","repair":false,"outputs":3,"x":580,"y":780,"wires":[["f1e954fd.3c21d8"],["65928861.c90a48"],["a259a26c.a21"]],"outputLabels":["Creation","Update","Delete"]},{"id":"f1e954fd.3c21d8","type":"switch","z":"dca608c3.7d8af8","name":"Creation","property":"payload.objectType","propertyType":"msg","rules":[{"t":"eq","v":"case","vt":"str"},{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"case_task","vt":"str"},{"t":"eq","v":"case_task_log","vt":"str"},{"t":"eq","v":"case_artifact_job","vt":"str"},{"t":"eq","v":"alert","vt":"str"},{"t":"eq","v":"user","vt":"str"}],"checkall":"false","repair":false,"outputs":7,"x":900,"y":480,"wires":[["e88b4cc2.f6afe"],["8c54e39.a1b4f2"],["64203fe8.e0ad5"],["3511de51.889a02"],["14544a8b.b6b2f5"],["44c595a4.45d45c"],["3eb4bedf.6e20a2"]],"inputLabels":["Operation"],"outputLabels":["case","case_artifact","case_task","case_task_log","action","alert","user"],"info":"No webhook data is received for the following events:\n\n- Creation of Dashboard\n- Creation of Case Templates\n"},{"id":"65928861.c90a48","type":"switch","z":"dca608c3.7d8af8","name":"Update","property":"payload.objectType","propertyType":"msg","rules":[{"t":"eq","v":"case","vt":"str"},{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"case_artifact_job","vt":"str"},{"t":"eq","v":"case_task","vt":"str"},{"t":"eq","v":"case_task_log","vt":"str"},{"t":"eq","v":"alert","vt":"str"},{"t":"eq","v":"user","vt":"str"}],"checkall":"false","repair":false,"outputs":7,"x":900,"y":860,"wires":[["eebe1748.1cd348"],["d703adc0.12fd1"],["2b738415.408d4c"],["6d97371a.406348"],["4ae621e1.9ae6"],["5786cee2.98109"],["54077728.447648"]],"inputLabels":["Operation"],"outputLabels":["case","case_artifact",null,"case_task","case_task_log","alert","user"]},{"id":"a259a26c.a21","type":"switch","z":"dca608c3.7d8af8","name":"Delete","property":"payload.objectType","propertyType":"msg","rules":[{"t":"eq","v":"case","vt":"str"},{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"case_task_log","vt":"str"}],"checkall":"false","repair":false,"outputs":3,"x":890,"y":1200,"wires":[["60c8bcfb.eff1f4"],["df708bab.348308"],["e9a8650c.e20cc8"]],"outputLabels":["case","case_artifact",""],"info":"Deleting a case task doesnt actually trigger a delete event. It triggers an `update` event where the status = cancelled"},{"id":"54077728.447648","type":"switch","z":"dca608c3.7d8af8","name":"User","property":"payload.object.status","propertyType":"msg","rules":[{"t":"eq","v":"Locked","vt":"str"},{"t":"eq","v":"Ok","vt":"str"}],"checkall":"false","repair":false,"outputs":2,"x":1130,"y":980,"wires":[["9429d6c5.5ac788"],["4e3e091c.d35388"]]},{"id":"9429d6c5.5ac788","type":"function","z":"dca608c3.7d8af8","name":"status: Locked","func":"msg.topic = \"[The Hive] A user account was locked\";\nmsg.from = \"from@example.com\";\nmsg.to = \"to@example.com\";\nreturn msg;","outputs":1,"noerr":0,"x":1380,"y":972,"wires":[[]],"info":"- User account was locked"},{"id":"4e3e091c.d35388","type":"function","z":"dca608c3.7d8af8","name":"status: Ok","func":"msg.topic = \"[The Hive] A user account was changed\";\nmsg.from = \"from@example.com\";\nmsg.to = \"to@example.com\";\nreturn msg;","outputs":1,"noerr":0,"x":1360,"y":1020,"wires":[[]],"info":"- User account was unlocked\n- User description was changed\n- User role was changed\n- User API key was added\n- User API key was revoked\n"},{"id":"485f3be.1ffcfc4","type":"function","z":"dca608c3.7d8af8","name":"status: Open","func":"// Fires when a Case is updated AND status = open\n// This can include things like TLP/PAP changes\n\nreturn msg;","outputs":1,"noerr":0,"x":1370,"y":660,"wires":[[]]},{"id":"eebe1748.1cd348","type":"switch","z":"dca608c3.7d8af8","name":"case","property":"payload.object.status","propertyType":"msg","rules":[{"t":"eq","v":"Open","vt":"str"}],"checkall":"true","repair":false,"outputs":1,"x":1130,"y":740,"wires":[["485f3be.1ffcfc4","e4b7b4bf.2fb828"]],"info":"- A case was modified"},{"id":"8c54e39.a1b4f2","type":"switch","z":"dca608c3.7d8af8","name":"case_artifact: Run Analyzer","property":"payload.object.dataType","propertyType":"msg","rules":[{"t":"eq","v":"ip","vt":"str"},{"t":"eq","v":"domain","vt":"str"}],"checkall":"true","repair":false,"outputs":2,"x":1600,"y":340,"wires":[["eb8cfeb7.a7118","a5dd8a8a.065b88"],["eb8cfeb7.a7118","a5dd8a8a.065b88"]],"info":"# References\n\n\n"},{"id":"2fce29bb.1b1376","type":"function","z":"dca608c3.7d8af8","name":"Add headers","func":"msg.thehive_url = 'https://{{ MANAGERIP }}/thehive';\nmsg.cortex_url = 'https://{{ MANAGERIP }}/cortex';\nmsg.cortex_id = 'CORTEX-SERVER-ID';\nreturn msg;","outputs":1,"noerr":0,"x":350,"y":780,"wires":[["a4ecb84a.805958"]]},{"id":"e4b7b4bf.2fb828","type":"function","z":"dca608c3.7d8af8","name":"status: Resolved","func":"// Fires when a case is closed (resolved)\n\nreturn msg;","outputs":1,"noerr":0,"x":1390,"y":720,"wires":[[]]},{"id":"e88b4cc2.f6afe","type":"function","z":"dca608c3.7d8af8","name":"case","func":"// Fires when a case is created\n// or when a responder is generated against a case\n\nreturn msg;","outputs":1,"noerr":0,"x":1130,"y":320,"wires":[[]]},{"id":"64203fe8.e0ad5","type":"function","z":"dca608c3.7d8af8","name":"case_task","func":"// Fires when a case task is created\nreturn msg;","outputs":1,"noerr":0,"x":1140,"y":400,"wires":[[]]},{"id":"3511de51.889a02","type":"function","z":"dca608c3.7d8af8","name":"case_task_log","func":"// Fires when a case task log is created\n\nreturn msg;","outputs":1,"noerr":0,"x":1163,"y":440,"wires":[[]]},{"id":"14544a8b.b6b2f5","type":"function","z":"dca608c3.7d8af8","name":"case_artifact_job","func":"// Fires when a Responder or Analyzser is Run on an existing observable\n\nreturn msg;","outputs":1,"noerr":0,"x":1173,"y":480,"wires":[[]]},{"id":"2b738415.408d4c","type":"function","z":"dca608c3.7d8af8","name":"case_artifact_job","func":"\nreturn msg;","outputs":1,"noerr":0,"x":1170,"y":820,"wires":[[]]},{"id":"3eb4bedf.6e20a2","type":"function","z":"dca608c3.7d8af8","name":"user","func":"// Fires when a user is created\n\nreturn msg;","outputs":1,"noerr":0,"x":1133,"y":560,"wires":[[]]},{"id":"d703adc0.12fd1","type":"function","z":"dca608c3.7d8af8","name":"case_artifact","func":"// Fires when an artifact is updated\nreturn msg;","outputs":1,"noerr":0,"x":1150,"y":780,"wires":[[]]},{"id":"6d97371a.406348","type":"function","z":"dca608c3.7d8af8","name":"case_task","func":"// Fires when a case task is updated\nreturn msg;","outputs":1,"noerr":0,"x":1140,"y":860,"wires":[[]]},{"id":"4ae621e1.9ae6","type":"function","z":"dca608c3.7d8af8","name":"case_task_log","func":"//Fires when a case_task_log is updated\n\nreturn msg;","outputs":1,"noerr":0,"x":1160,"y":900,"wires":[[]]},{"id":"60c8bcfb.eff1f4","type":"function","z":"dca608c3.7d8af8","name":"case","func":"//Fires when a case is deleted\nreturn msg;","outputs":1,"noerr":0,"x":1130,"y":1160,"wires":[[]]},{"id":"df708bab.348308","type":"function","z":"dca608c3.7d8af8","name":"case_artifact","func":"//Fires when a case_artifact is deleted\nreturn msg;","outputs":1,"noerr":0,"x":1150,"y":1200,"wires":[[]]},{"id":"e9a8650c.e20cc8","type":"function","z":"dca608c3.7d8af8","name":"case_task_log","func":"//Fires when a case_task_log is deleted\nreturn msg;","outputs":1,"noerr":0,"x":1160,"y":1240,"wires":[[]]},{"id":"5786cee2.98109","type":"function","z":"dca608c3.7d8af8","name":"alert","func":"//Fires when an alert is updated\nreturn msg;","outputs":1,"noerr":0,"x":1130,"y":940,"wires":[[]]},{"id":"44c595a4.45d45c","type":"change","z":"dca608c3.7d8af8","d":true,"name":"Convert Alert Msg to Artifacts","rules":[{"t":"move","p":"payload.object.artifacts","pt":"msg","to":"payload","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":1200,"y":520,"wires":[["6dcca25e.04bd2c"]]},{"id":"6dcca25e.04bd2c","type":"split","z":"dca608c3.7d8af8","name":"Split Artifacts","splt":"\\n","spltType":"str","arraySplt":1,"arraySpltType":"len","stream":false,"addname":"","x":1430,"y":520,"wires":[["767c84f2.c9ba2c"]]},{"id":"767c84f2.c9ba2c","type":"switch","z":"dca608c3.7d8af8","name":"alert: Run Analyzer","property":"payload.dataType","propertyType":"msg","rules":[{"t":"eq","v":"ip","vt":"str"},{"t":"eq","v":"domain","vt":"str"}],"checkall":"true","repair":false,"outputs":2,"x":1630,"y":400,"wires":[["eb8cfeb7.a7118","a5dd8a8a.065b88"],["a5dd8a8a.065b88","eb8cfeb7.a7118"]],"info":"# References\n\n\n"},{"id":"82ad0f08.7a53f","type":"http response","z":"dca608c3.7d8af8","name":"Ack Event Receipt","statusCode":"200","headers":{},"x":250,"y":940,"wires":[]},{"id":"a5dd8a8a.065b88","type":"function","z":"dca608c3.7d8af8","name":"Run Analyzer: CERT DNS","func":"msg.analyzer_id = \"4f28afc20d78f98df425e36e561af33f\";\n\nif (msg.payload.objectId) {\n msg.tag = \"case_artifact\"\n msg.artifact_id = msg.payload.objectId\n msg.url = msg.thehive_url + '/api/connector/cortex/job';\n msg.payload = {\n 'cortexId' : msg.cortex_id,\n 'artifactId': msg.artifact_id,\n 'analyzerId': msg.analyzer_id\n };\n}\nelse {\n msg.tag = \"observable\"\n msg.observable = msg.payload.data\n msg.dataType = msg.payload.dataType\n\n msg.url = msg.cortex_url + '/api/analyzer/' + msg.analyzer_id + '/run';\n msg.payload = {\n 'data' : msg.observable,\n 'dataType': msg.dataType \n };\n}\nreturn msg;","outputs":1,"noerr":0,"x":1930,"y":420,"wires":[["f050a09f.b2201"]]},{"id":"eb8cfeb7.a7118","type":"function","z":"dca608c3.7d8af8","name":"Run Analyzer: Urlscan","func":"msg.analyzer_id = \"54e51b62c6c8ddc3cbc3cbdd889a0557\";\n\nif (msg.payload.objectId) {\n msg.tag = \"case_artifact\"\n msg.artifact_id = msg.payload.objectId\n msg.url = msg.thehive_url + '/api/connector/cortex/job';\n msg.payload = {\n 'cortexId' : msg.cortex_id,\n 'artifactId': msg.artifact_id,\n 'analyzerId': msg.analyzer_id\n };\n}\nelse {\n msg.tag = \"observable\"\n msg.observable = msg.payload.data\n msg.dataType = msg.payload.dataType\n\n msg.url = msg.cortex_url + '/api/analyzer/' + msg.analyzer_id + '/run';\n msg.payload = {\n 'data' : msg.observable,\n 'dataType': msg.dataType \n };\n}\nreturn msg;","outputs":1,"noerr":0,"x":1920,"y":320,"wires":[["f050a09f.b2201"]]},{"id":"1c448528.3032fb","type":"http request","z":"dca608c3.7d8af8","name":"Submit to Cortex","method":"POST","ret":"obj","paytoqs":false,"url":"","tls":"4db74fa6.2556d","persist":false,"proxy":"","authType":"bearer","credentials": {"user": "", "password": "{{ CORTEXKEY }}"},"x":2450,"y":420,"wires":[["ea6614fb.752a78"]]},{"id":"ea6614fb.752a78","type":"debug","z":"dca608c3.7d8af8","name":"Debug","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","x":2670,"y":360,"wires":[]},{"id":"f050a09f.b2201","type":"switch","z":"dca608c3.7d8af8","name":"Cases vs Alerts","property":"tag","propertyType":"msg","rules":[{"t":"eq","v":"case_artifact","vt":"str"},{"t":"eq","v":"observable","vt":"str"}],"checkall":"true","repair":false,"outputs":2,"x":2200,"y":360,"wires":[["f7fca977.a73b28"],["1c448528.3032fb"]],"inputLabels":["Data"],"outputLabels":["Cases","Alerts"]},{"id":"f7fca977.a73b28","type":"http request","z":"dca608c3.7d8af8","name":"Submit to TheHive","method":"POST","ret":"obj","paytoqs":false,"url":"","tls":"4db74fa6.2556d","persist":false,"proxy":"","authType":"bearer","credentials": {"user": "", "password": "{{ HIVEKEY }}"},"x":2450,"y":280,"wires":[["ea6614fb.752a78"]]}] diff --git a/salt/nodered/init.sls b/salt/nodered/init.sls index bec8f266a..34aacbd81 100644 --- a/salt/nodered/init.sls +++ b/salt/nodered/init.sls @@ -13,7 +13,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} # Create the nodered group noderedgroup: diff --git a/salt/pcap/files/sensoroni.json b/salt/pcap/files/sensoroni.json index ab99c175c..79e97a75b 100644 --- a/salt/pcap/files/sensoroni.json +++ b/salt/pcap/files/sensoroni.json @@ -1,5 +1,5 @@ {%- set MANAGER = salt['grains.get']('master') -%} -{%- set SENSORONIKEY = salt['pillar.get']('static:sensoronikey', '') -%} +{%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') -%} {%- set CHECKININTERVALMS = salt['pillar.get']('pcap:sensor_checkin_interval_ms', 10000) -%} { "logFilename": "/opt/sensoroni/logs/sensoroni.log", diff --git a/salt/pcap/init.sls b/salt/pcap/init.sls index 1a9de6611..3db7a227c 100644 --- a/salt/pcap/init.sls +++ b/salt/pcap/init.sls @@ -12,8 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set INTERFACE = salt['pillar.get']('sensor:interface', 'bond0') %} {% set BPF_STENO = salt['pillar.get']('steno:bpf', None) %} diff --git a/salt/playbook/init.sls b/salt/playbook/init.sls index 44b806f9a..d390a36fb 100644 --- a/salt/playbook/init.sls +++ b/salt/playbook/init.sls @@ -1,6 +1,6 @@ {% set MANAGERIP = salt['pillar.get']('manager:mainip', '') %} -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set MAINIP = salt['grains.get']('ip_interfaces').get(salt['pillar.get']('sensor:mainint', salt['pillar.get']('manager:mainint', salt['pillar.get']('elasticsearch:mainint', salt['pillar.get']('host:mainint')))))[0] %} {%- set MYSQLPASS = salt['pillar.get']('secrets:mysql', None) -%} diff --git a/salt/reactor/fleet.sls b/salt/reactor/fleet.sls index 177dabf3a..4e4e13791 100644 --- a/salt/reactor/fleet.sls +++ b/salt/reactor/fleet.sls @@ -10,7 +10,7 @@ def run(): MINIONID = data['id'] ACTION = data['data']['action'] LOCAL_SALT_DIR = "/opt/so/saltstack/local" - STATICFILE = f"{LOCAL_SALT_DIR}/pillar/static.sls" + STATICFILE = f"{LOCAL_SALT_DIR}/pillar/global.sls" SECRETSFILE = f"{LOCAL_SALT_DIR}/pillar/secrets.sls" if MINIONID.split('_')[-1] in ['manager','eval','fleet','managersearch','standalone']: diff --git a/salt/redis/init.sls b/salt/redis/init.sls index 5a981e688..4864fc8a2 100644 --- a/salt/redis/init.sls +++ b/salt/redis/init.sls @@ -12,8 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} # Redis Setup diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 31e49fc86..b9470652b 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -1,5 +1,5 @@ -{%- set MANAGERIP = salt['pillar.get']('static:managerip', '') -%} -{%- set SENSORONIKEY = salt['pillar.get']('static:sensoronikey', '') -%} +{%- set MANAGERIP = salt['pillar.get']('global:managerip', '') -%} +{%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') -%} { "logFilename": "/opt/sensoroni/logs/sensoroni-server.log", "server": { diff --git a/salt/soc/init.sls b/salt/soc/init.sls index e3fdf538a..1c25f42a1 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -1,5 +1,5 @@ -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} socdir: diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index 477113376..093b4fd3e 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -1,6 +1,6 @@ {%- set MANAGER = salt['pillar.get']('manager:url_base', '') %} -{%- set HIVEKEY = salt['pillar.get']('static:hivekey', '') %} -{%- set CORTEXKEY = salt['pillar.get']('static:cortexorguserkey', '') %} +{%- set HIVEKEY = salt['pillar.get']('global:hivekey', '') %} +{%- set CORTEXKEY = salt['pillar.get']('global:cortexorguserkey', '') %} [es] es_url = http://{{MANAGER}}:9200 diff --git a/salt/soctopus/files/templates/es-generic.template b/salt/soctopus/files/templates/es-generic.template index b56050741..8183a5af4 100644 --- a/salt/soctopus/files/templates/es-generic.template +++ b/salt/soctopus/files/templates/es-generic.template @@ -1,4 +1,4 @@ -{% set ES = salt['pillar.get']('static:managerip', '') %} +{% set ES = salt['pillar.get']('global:managerip', '') %} alert: modules.so.playbook-es.PlaybookESAlerter elasticsearch_host: "{{ ES }}:9200" diff --git a/salt/soctopus/files/templates/generic.template b/salt/soctopus/files/templates/generic.template index 7bb5a969d..cdd5947d3 100644 --- a/salt/soctopus/files/templates/generic.template +++ b/salt/soctopus/files/templates/generic.template @@ -1,6 +1,6 @@ -{% set es = salt['pillar.get']('static:managerip', '') %} -{% set hivehost = salt['pillar.get']('static:managerip', '') %} -{% set hivekey = salt['pillar.get']('static:hivekey', '') %} +{% set es = salt['pillar.get']('global:managerip', '') %} +{% set hivehost = salt['pillar.get']('global:managerip', '') %} +{% set hivekey = salt['pillar.get']('global:hivekey', '') %} alert: hivealerter hive_connection: diff --git a/salt/soctopus/files/templates/osquery.template b/salt/soctopus/files/templates/osquery.template index 4fff9a1d5..352c3d69a 100644 --- a/salt/soctopus/files/templates/osquery.template +++ b/salt/soctopus/files/templates/osquery.template @@ -1,6 +1,6 @@ -{% set es = salt['pillar.get']('static:managerip', '') %} -{% set hivehost = salt['pillar.get']('static:managerip', '') %} -{% set hivekey = salt['pillar.get']('static:hivekey', '') %} +{% set es = salt['pillar.get']('global:managerip', '') %} +{% set hivehost = salt['pillar.get']('global:managerip', '') %} +{% set hivekey = salt['pillar.get']('global:hivekey', '') %} alert: hivealerter hive_connection: diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls index 3fcdf8717..7526974df 100644 --- a/salt/soctopus/init.sls +++ b/salt/soctopus/init.sls @@ -1,8 +1,8 @@ -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {%- set MANAGER_URL = salt['pillar.get']('manager:url_base', '') %} -{%- set MANAGER_IP = salt['pillar.get']('static:managerip', '') %} +{%- set MANAGER_IP = salt['pillar.get']('global:managerip', '') %} soctopusdir: file.directory: diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index dfbd4c12a..1cef1bf0a 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -1,11 +1,11 @@ {% set manager = salt['grains.get']('master') %} -{% set managerip = salt['pillar.get']('static:managerip', '') %} +{% set managerip = salt['pillar.get']('global:managerip', '') %} {% set HOSTNAME = salt['grains.get']('host') %} {% set global_ca_text = [] %} {% set global_ca_server = [] %} {% set MAININT = salt['pillar.get']('host:mainint') %} {% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} -{% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('static:fleet_custom_hostname', None) %} +{% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('global:fleet_custom_hostname', None) %} {% if grains.id.split('_')|last in ['manager', 'eval', 'standalone'] %} {% set trusttheca_text = salt['mine.get'](grains.id, 'x509.get_pem_entries')[grains.id]['/etc/pki/ca.crt']|replace('\n', '') %} diff --git a/salt/strelka/files/backend/backend.yaml b/salt/strelka/files/backend/backend.yaml index b25e5630d..8748a4fd6 100644 --- a/salt/strelka/files/backend/backend.yaml +++ b/salt/strelka/files/backend/backend.yaml @@ -2,7 +2,7 @@ {%- set mainint = salt['pillar.get']('sensor:mainint') %} {%- set ip = salt['grains.get']('ip_interfaces:' ~ mainint[0], salt['pillar.get']('sensor:mainip')) %} {%- else %} - {%- set ip = salt['pillar.get']('static:managerip') %} + {%- set ip = salt['pillar.get']('global:managerip') %} {%- endif -%} logging_cfg: '/etc/strelka/logging.yaml' limits: diff --git a/salt/strelka/files/filestream/filestream.yaml b/salt/strelka/files/filestream/filestream.yaml index 539e4314c..1dc6795d9 100644 --- a/salt/strelka/files/filestream/filestream.yaml +++ b/salt/strelka/files/filestream/filestream.yaml @@ -2,7 +2,7 @@ {%- set mainint = salt['pillar.get']('sensor:mainint') %} {%- set ip = salt['grains.get']('ip_interfaces:' ~ mainint[0], salt['pillar.get']('sensor:mainip')) %} {%- else %} - {%- set ip = salt['pillar.get']('static:managerip') %} + {%- set ip = salt['pillar.get']('global:managerip') %} {%- endif -%} conn: server: '{{ ip }}:57314' diff --git a/salt/strelka/files/frontend/frontend.yaml b/salt/strelka/files/frontend/frontend.yaml index 5d72f1e0d..23edef3e3 100644 --- a/salt/strelka/files/frontend/frontend.yaml +++ b/salt/strelka/files/frontend/frontend.yaml @@ -2,7 +2,7 @@ {%- set mainint = salt['pillar.get']('sensor:mainint') %} {%- set ip = salt['grains.get']('ip_interfaces:' ~ mainint[0], salt['pillar.get']('sensor:mainip')) %} {%- else %} - {%- set ip = salt['pillar.get']('static:managerip') %} + {%- set ip = salt['pillar.get']('global:managerip') %} {%- endif -%} server: ":57314" coordinator: diff --git a/salt/strelka/files/manager/manager.yaml b/salt/strelka/files/manager/manager.yaml index db9dd7f91..b4a73b1c0 100644 --- a/salt/strelka/files/manager/manager.yaml +++ b/salt/strelka/files/manager/manager.yaml @@ -2,7 +2,7 @@ {%- set mainint = salt['pillar.get']('sensor:mainint') %} {%- set ip = salt['grains.get']('ip_interfaces:' ~ mainint[0], salt['pillar.get']('sensor:mainip')) %} {%- else %} - {%- set ip = salt['pillar.get']('static:managerip') %} + {%- set ip = salt['pillar.get']('global:managerip') %} {%- endif -%} coordinator: addr: '{{ ip }}:6380' diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index c6a900e8e..e85b62f83 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -13,9 +13,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . {%- set MANAGER = salt['grains.get']('master') %} -{%- set MANAGERIP = salt['pillar.get']('static:managerip', '') %} -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {%- set STRELKA_RULES = salt['pillar.get']('strelka:rules', '1') -%} # Strelka config diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index c0677db16..783f174ca 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -14,9 +14,9 @@ # along with this program. If not, see . {% set interface = salt['pillar.get']('sensor:interface', 'bond0') %} -{% set ZEEKVER = salt['pillar.get']('static:zeekversion', '') %} -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set ZEEKVER = salt['pillar.get']('global:zeekversion', '') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set BPF_NIDS = salt['pillar.get']('nids:bpf') %} {% set BPF_STATUS = 0 %} diff --git a/salt/suricata/suricata_config.map.jinja b/salt/suricata/suricata_config.map.jinja index 9fb3c9a7f..a544f6d96 100644 --- a/salt/suricata/suricata_config.map.jinja +++ b/salt/suricata/suricata_config.map.jinja @@ -11,7 +11,7 @@ HOME_NET: "[{{salt['pillar.get']('sensor:hnsensor')}}]" {% endload %} {% else %} {% load_yaml as homenet %} -HOME_NET: "[{{salt['pillar.get']('static:hnmanager', '')}}]" +HOME_NET: "[{{salt['pillar.get']('global:hnmanager', '')}}]" {% endload %} {% endif %} @@ -44,7 +44,7 @@ HOME_NET: "[{{salt['pillar.get']('static:hnmanager', '')}}]" {% endfor %} {% set surimeta_evelog_index = surimeta_evelog_index[0] %} -{% if salt['pillar.get']('static:zeekversion', 'ZEEK') == 'SURICATA' %} +{% if salt['pillar.get']('global:zeekversion', 'ZEEK') == 'SURICATA' %} {% do suricata_defaults.suricata.config.outputs[default_evelog_index]['eve-log'].types.extend(suricata_meta.suricata.config.outputs[surimeta_evelog_index]['eve-log'].types) %} {% endif %} diff --git a/salt/tcpreplay/init.sls b/salt/tcpreplay/init.sls index 7247e4505..a828c72f1 100644 --- a/salt/tcpreplay/init.sls +++ b/salt/tcpreplay/init.sls @@ -1,6 +1,6 @@ {% if grains['role'] == 'so-sensor' or grains['role'] == 'so-eval' %} -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} so-tcpreplay: diff --git a/salt/telegraf/init.sls b/salt/telegraf/init.sls index 668a8839a..c252cdb5b 100644 --- a/salt/telegraf/init.sls +++ b/salt/telegraf/init.sls @@ -1,6 +1,6 @@ {% set MANAGER = salt['grains.get']('master') %} -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} # Add Telegraf to monitor all the things. tgraflogdir: diff --git a/salt/thehive/etc/application.conf b/salt/thehive/etc/application.conf index 8aaf7a9a5..675c5222c 100644 --- a/salt/thehive/etc/application.conf +++ b/salt/thehive/etc/application.conf @@ -1,6 +1,6 @@ -{%- set MANAGERIP = salt['pillar.get']('static:managerip', '') %} -{%- set CORTEXKEY = salt['pillar.get']('static:cortexorguserkey', '') %} -{%- set HIVEPLAYSECRET = salt['pillar.get']('static:hiveplaysecret', '') %} +{%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %} +{%- set CORTEXKEY = salt['pillar.get']('global:cortexorguserkey', '') %} +{%- set HIVEPLAYSECRET = salt['pillar.get']('global:hiveplaysecret', '') %} # Secret Key # The secret key is used to secure cryptographic functions. diff --git a/salt/thehive/etc/cortex-application.conf b/salt/thehive/etc/cortex-application.conf index c7e52d954..d84566068 100644 --- a/salt/thehive/etc/cortex-application.conf +++ b/salt/thehive/etc/cortex-application.conf @@ -1,5 +1,5 @@ -{%- set MANAGERIP = salt['pillar.get']('static:managerip', '') %} -{%- set CORTEXPLAYSECRET = salt['pillar.get']('static:cortexplaysecret', '') %} +{%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %} +{%- set CORTEXPLAYSECRET = salt['pillar.get']('global:cortexplaysecret', '') %} # Secret Key # The secret key is used to secure cryptographic functions. diff --git a/salt/thehive/init.sls b/salt/thehive/init.sls index 062637855..ffbb50f0c 100644 --- a/salt/thehive/init.sls +++ b/salt/thehive/init.sls @@ -1,6 +1,6 @@ {% set MANAGERIP = salt['pillar.get']('manager:mainip', '') %} -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} thehiveconfdir: file.directory: diff --git a/salt/thehive/scripts/cortex_init b/salt/thehive/scripts/cortex_init index 7eb50df5e..6f5d890ae 100644 --- a/salt/thehive/scripts/cortex_init +++ b/salt/thehive/scripts/cortex_init @@ -1,18 +1,18 @@ #!/bin/bash -# {%- set MANAGERIP = salt['pillar.get']('static:managerip', '') %} -# {%- set CORTEXUSER = salt['pillar.get']('static:cortexuser', 'cortexadmin') %} -# {%- set CORTEXPASSWORD = salt['pillar.get']('static:cortexpassword', 'cortexchangeme') %} -# {%- set CORTEXKEY = salt['pillar.get']('static:cortexkey', '') %} -# {%- set CORTEXORGNAME = salt['pillar.get']('static:cortexorgname', '') %} -# {%- set CORTEXORGUSER = salt['pillar.get']('static:cortexorguser', 'soadmin') %} -# {%- set CORTEXORGUSERKEY = salt['pillar.get']('static:cortexorguserkey', '') %} +# {%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %} +# {%- set CORTEXUSER = salt['pillar.get']('global:cortexuser', 'cortexadmin') %} +# {%- set CORTEXPASSWORD = salt['pillar.get']('global:cortexpassword', 'cortexchangeme') %} +# {%- set CORTEXKEY = salt['pillar.get']('global:cortexkey', '') %} +# {%- set CORTEXORGNAME = salt['pillar.get']('global:cortexorgname', '') %} +# {%- set CORTEXORGUSER = salt['pillar.get']('global:cortexorguser', 'soadmin') %} +# {%- set CORTEXORGUSERKEY = salt['pillar.get']('global:cortexorguserkey', '') %} default_salt_dir=/opt/so/saltstack/default cortex_clean(){ - sed -i '/^ cortexuser:/d' /opt/so/saltstack/local/pillar/static.sls - sed -i '/^ cortexpassword:/d' /opt/so/saltstack/local/pillar/static.sls - sed -i '/^ cortexorguser:/d' /opt/so/saltstack/local/pillar/static.sls + sed -i '/^ cortexuser:/d' /opt/so/saltstack/local/pillar/global.sls + sed -i '/^ cortexpassword:/d' /opt/so/saltstack/local/pillar/global.sls + sed -i '/^ cortexorguser:/d' /opt/so/saltstack/local/pillar/global.sls } cortex_init(){ diff --git a/salt/thehive/scripts/hive_init b/salt/thehive/scripts/hive_init index 0caff6e2d..c44af6339 100755 --- a/salt/thehive/scripts/hive_init +++ b/salt/thehive/scripts/hive_init @@ -1,12 +1,12 @@ #!/bin/bash -# {%- set MANAGERIP = salt['pillar.get']('static:managerip', '') %} -# {%- set THEHIVEUSER = salt['pillar.get']('static:hiveuser', 'hiveadmin') %} -# {%- set THEHIVEPASSWORD = salt['pillar.get']('static:hivepassword', 'hivechangeme') %} -# {%- set THEHIVEKEY = salt['pillar.get']('static:hivekey', '') %} +# {%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %} +# {%- set THEHIVEUSER = salt['pillar.get']('global:hiveuser', 'hiveadmin') %} +# {%- set THEHIVEPASSWORD = salt['pillar.get']('global:hivepassword', 'hivechangeme') %} +# {%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %} thehive_clean(){ - sed -i '/^ hiveuser:/d' /opt/so/saltstack/local/pillar/static.sls - sed -i '/^ hivepassword:/d' /opt/so/saltstack/local/pillar/static.sls + sed -i '/^ hiveuser:/d' /opt/so/saltstack/local/pillar/global.sls + sed -i '/^ hivepassword:/d' /opt/so/saltstack/local/pillar/global.sls } thehive_init(){ diff --git a/salt/top.sls b/salt/top.sls index ff2fbfb0e..30f198b05 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -1,11 +1,11 @@ -{%- set ZEEKVER = salt['pillar.get']('static:zeekversion', '') -%} -{%- set WAZUH = salt['pillar.get']('static:wazuh', '0') -%} +{%- set ZEEKVER = salt['pillar.get']('global:zeekversion', '') -%} +{%- set WAZUH = salt['pillar.get']('global:wazuh', '0') -%} {%- set THEHIVE = salt['pillar.get']('manager:thehive', '0') -%} {%- set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') -%} {%- set FREQSERVER = salt['pillar.get']('manager:freq', '0') -%} {%- set DOMAINSTATS = salt['pillar.get']('manager:domainstats', '0') -%} -{%- set FLEETMANAGER = salt['pillar.get']('static:fleet_manager', False) -%} -{%- set FLEETNODE = salt['pillar.get']('static:fleet_node', False) -%} +{%- set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) -%} +{%- set FLEETNODE = salt['pillar.get']('global:fleet_node', False) -%} {%- set STRELKA = salt['pillar.get']('strelka:enabled', '0') -%} {% import_yaml 'salt/minion.defaults.yaml' as salt %} {% set saltversion = salt.salt.minion.version %} diff --git a/salt/wazuh/files/agent/ossec.conf b/salt/wazuh/files/agent/ossec.conf index 8d38868ef..7e33f5599 100644 --- a/salt/wazuh/files/agent/ossec.conf +++ b/salt/wazuh/files/agent/ossec.conf @@ -1,5 +1,5 @@ {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} - {%- set ip = salt['pillar.get']('static:managerip', '') %} + {%- set ip = salt['pillar.get']('global:managerip', '') %} {%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} {%- set ip = salt['pillar.get']('elasticsearch:mainip', '') %} {%- elif grains['role'] == 'so-sensor' %} diff --git a/salt/wazuh/files/agent/wazuh-register-agent b/salt/wazuh/files/agent/wazuh-register-agent index bed0ba57f..c6411b492 100755 --- a/salt/wazuh/files/agent/wazuh-register-agent +++ b/salt/wazuh/files/agent/wazuh-register-agent @@ -1,5 +1,5 @@ {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} - {%- set ip = salt['pillar.get']('static:managerip', '') %} + {%- set ip = salt['pillar.get']('global:managerip', '') %} {%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} {%- set ip = salt['pillar.get']('elasticsearch:mainip', '') %} {%- elif grains['role'] == 'so-sensor' %} diff --git a/salt/wazuh/files/wazuh-manager-whitelist b/salt/wazuh/files/wazuh-manager-whitelist index 8a8bc9832..c3ecf31a9 100755 --- a/salt/wazuh/files/wazuh-manager-whitelist +++ b/salt/wazuh/files/wazuh-manager-whitelist @@ -1,5 +1,5 @@ -{%- set MANAGERIP = salt['pillar.get']('static:managerip', '') %} -{%- set WAZUH_ENABLED = salt['pillar.get']('static:wazuh', '0') %} +{%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %} +{%- set WAZUH_ENABLED = salt['pillar.get']('global:wazuh', '0') %} #!/bin/bash local_salt_dir=/opt/so/saltstack/local diff --git a/salt/wazuh/init.sls b/salt/wazuh/init.sls index 09c4e258b..94b16b199 100644 --- a/salt/wazuh/init.sls +++ b/salt/wazuh/init.sls @@ -1,6 +1,6 @@ {%- set HOSTNAME = salt['grains.get']('host', '') %} -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} # Add ossec group ossecgroup: diff --git a/salt/yum/etc/yum.conf.jinja b/salt/yum/etc/yum.conf.jinja index aab63550b..22449083e 100644 --- a/salt/yum/etc/yum.conf.jinja +++ b/salt/yum/etc/yum.conf.jinja @@ -11,6 +11,6 @@ installonly_limit={{ salt['pillar.get']('yum:config:installonly_limit', 2) }} bugtracker_url=http://bugs.centos.org/set_project.php?project_id=23&ref=http://bugs.centos.org/bug_report_page.php?category=yum distroverpkg=centos-release -{% if salt['pillar.get']('static:managerupdate', '0') %} +{% if salt['pillar.get']('global:managerupdate', '0') %} proxy=http://{{ salt['pillar.get']('yum:config:proxy', salt['config.get']('master')) }}:3142 {% endif %} \ No newline at end of file diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index 68908a2ce..8743878da 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -1,5 +1,5 @@ -{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} -{% set IMAGEREPO = salt['pillar.get']('static:imagerepo') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set BPF_ZEEK = salt['pillar.get']('zeek:bpf', {}) %} {% set BPF_STATUS = 0 %} diff --git a/setup/so-functions b/setup/so-functions index ad4b4252f..7ebfe3f7a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1006,8 +1006,8 @@ manager_pillar() { cat "$pillar_file" >> "$setup_log" 2>&1 } -manager_static() { - local static_pillar="$local_salt_dir/pillar/static.sls" +manager_global() { + local global_pillar="$local_salt_dir/pillar/global.sls" if [ -z "$SENSOR_CHECKIN_INTERVAL_MS" ]; then SENSOR_CHECKIN_INTERVAL_MS=10000 @@ -1016,9 +1016,9 @@ manager_static() { fi fi - # Create a static file for global values + # Create a global file for global values printf '%s\n'\ - "static:"\ + "global:"\ " soversion: $SOVERSION"\ " hnmanager: $HNMANAGER"\ " ntpserver: $NTPSERVER"\ @@ -1117,10 +1117,13 @@ manager_static() { " shards: 5"\ " warm: 7"\ " close: 365"\ - " delete: 45" > "$static_pillar" - + " delete: 45"\ + "minio:"\ + " access_key: $ACCESS_KEY"\ + " access_secret: $ACCESS_SECRET" > "$global_pillar" + printf '%s\n' '----' >> "$setup_log" 2>&1 - cat "$static_pillar" >> "$setup_log" 2>&1 + cat "$global_pillar" >> "$setup_log" 2>&1 } minio_generate_keys() { @@ -1520,10 +1523,6 @@ sensor_pillar() { if [ "$HNSENSOR" != 'inherit' ]; then echo " hnsensor: $HNSENSOR" >> "$pillar_file" fi - printf '%s\n'\ - " access_key: $ACCESS_KEY"\ - " access_secret: $ACCESS_SECRET"\ - "" >> "$pillar_file" printf '%s\n' '----' >> "$setup_log" 2>&1 cat "$pillar_file" >> "$setup_log" 2>&1 diff --git a/setup/so-setup b/setup/so-setup index 68ca99824..7335b5acc 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -428,8 +428,8 @@ fi set_progress_str 11 'Updating sudoers file for soremote user' update_sudoers >> $setup_log 2>&1 - set_progress_str 12 'Generating manager static pillar' - manager_static >> $setup_log 2>&1 + set_progress_str 12 'Generating manager global pillar' + manager_global >> $setup_log 2>&1 set_progress_str 13 'Generating manager pillar' manager_pillar >> $setup_log 2>&1 @@ -571,7 +571,7 @@ fi if [[ $is_fleet_standalone && $FLEETCUSTOMHOSTNAME != '' ]]; then set_progress_str 77 "$(print_salt_state_apply 'fleet.event_update-custom-hostname')" - pillar_override="{\"static\":{\"fleet_custom_hostname\": \"$FLEETCUSTOMHOSTNAME\"}}" + pillar_override="{\"global\":{\"fleet_custom_hostname\": \"$FLEETCUSTOMHOSTNAME\"}}" salt-call state.apply -l info fleet.event_update-custom-hostname pillar="$pillar_override" >> $setup_log 2>&1 fi From 407160b72989c5b5b7a3d16886389bc788137500 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 16:23:03 -0400 Subject: [PATCH 07/36] Update changes.json --- salt/soc/files/soc/changes.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/changes.json b/salt/soc/files/soc/changes.json index dc3e4118f..4f359a996 100644 --- a/salt/soc/files/soc/changes.json +++ b/salt/soc/files/soc/changes.json @@ -1,6 +1,10 @@ { - "title": "Security Onion 2.0.2 RC1 is here!", + "title": "Security Onion 2.0.3 RC1 is here!", "changes": [ + { "summary": "Resolved an issue with large drives and the ISO install." }, + { "summary": "Modified ISO installation to use Logical Volume Management (LVM) for disk partitioning." }, + { "summary": "Updated Elastic Stack components to version 7.8.1." }, + { "summary": "Updated Zeek to version 3.0.8." }, { "summary": "Fixed standalone pcap interval issue." }, { "summary": "Security Fix 1067: variables.txt from ISO install stays on disk for 10 days." }, { "summary": "Security Fix 1068: Remove user values from static.sls." }, From c56ead08e950867b4c644e946116ee0915577ce0 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 16:28:50 -0400 Subject: [PATCH 08/36] add so minio docker --- salt/common/tools/sbin/so-docker-refresh | 1 + salt/common/tools/sbin/soup | 1 + salt/minio/init.sls | 7 +++---- setup/so-functions | 1 + 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-docker-refresh b/salt/common/tools/sbin/so-docker-refresh index ace1e9554..770d9f241 100755 --- a/salt/common/tools/sbin/so-docker-refresh +++ b/salt/common/tools/sbin/so-docker-refresh @@ -76,6 +76,7 @@ if [ $MANAGERCHECK != 'so-helix' ]; then "so-kibana:$VERSION" \ "so-kratos:$VERSION" \ "so-logstash:$VERSION" \ + "so-minio:$VERSION" \ "so-mysql:$VERSION" \ "so-nginx:$VERSION" \ "so-pcaptools:$VERSION" \ diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 48d9314a3..608394530 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -111,6 +111,7 @@ update_dockers() { "so-kibana" \ "so-kratos" \ "so-logstash" \ + "so-minio" \ "so-mysql" \ "so-nginx" \ "so-pcaptools" \ diff --git a/salt/minio/init.sls b/salt/minio/init.sls index 438face99..f85effe09 100644 --- a/salt/minio/init.sls +++ b/salt/minio/init.sls @@ -15,6 +15,8 @@ {% set access_key = salt['pillar.get']('minio:access_key', '') %} {% set access_secret = salt['pillar.get']('minio:access_secret', '') %} +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} # Minio Setup minioconfdir: @@ -38,12 +40,9 @@ logstashbucket: - group: 939 - makedirs: True -minio/minio: - docker_image.present - minio: docker_container.running: - - image: minio/minio + - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-minio:{{ VERSION }} - hostname: so-minio - user: socore - port_bindings: diff --git a/setup/so-functions b/setup/so-functions index 7ebfe3f7a..de14447e4 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -789,6 +789,7 @@ docker_seed_registry() { "so-grafana:$VERSION" \ "so-influxdb:$VERSION" \ "so-kibana:$VERSION" \ + "so-minio:$VERSION" \ "so-mysql:$VERSION" \ "so-pcaptools:$VERSION" \ "so-playbook:$VERSION" \ From fd039b3008dac2c7dc3328731ba77aae3cd827dc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 17:11:20 -0400 Subject: [PATCH 09/36] Fix top file for minio --- salt/top.sls | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/salt/top.sls b/salt/top.sls index 30f198b05..34b825355 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -142,7 +142,6 @@ base: - manager - idstools - suricata.manager - - redis {%- if FLEETMANAGER or FLEETNODE or PLAYBOOK != 0 %} - mysql {%- endif %} @@ -151,6 +150,7 @@ base: {%- endif %} - logstash - minio + - redis - kibana - elastalert - filebeat @@ -159,6 +159,7 @@ base: {%- if FLEETMANAGER or FLEETNODE %} - fleet - fleet.install_package + - redis {%- endif %} - soctopus {%- if THEHIVE != 0 %} @@ -190,7 +191,6 @@ base: - idstools - suricata.manager - healthcheck - - redis {%- if FLEETMANAGER or FLEETNODE or PLAYBOOK != 0 %} - mysql {%- endif %} @@ -314,7 +314,7 @@ base: - manager - idstools - suricata.manager - - redis + - minio {%- if FLEETMANAGER or FLEETNODE or PLAYBOOK != 0 %} - mysql {%- endif %} @@ -330,6 +330,7 @@ base: - schedule {%- if FLEETMANAGER or FLEETNODE %} - fleet + - redis - fleet.install_package {%- endif %} - soctopus @@ -353,7 +354,7 @@ base: - common - telegraf - firewall - - redis + - minio {%- if WAZUH != 0 %} - wazuh {%- endif %} @@ -362,6 +363,7 @@ base: - filebeat {%- if FLEETMANAGER or FLEETNODE %} - fleet.install_package + - redis {%- endif %} - pcap - suricata From 9c5a969c2e18b96665ea21383a0f1f1ba5713811 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 17:18:09 -0400 Subject: [PATCH 10/36] Fix minio init --- salt/minio/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/minio/init.sls b/salt/minio/init.sls index f85effe09..d77c775aa 100644 --- a/salt/minio/init.sls +++ b/salt/minio/init.sls @@ -17,6 +17,8 @@ {% set access_secret = salt['pillar.get']('minio:access_secret', '') %} {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} +{% set MANAGER = salt['grains.get']('master') %} + # Minio Setup minioconfdir: From 38d0f519ce79e50418a272db9b12e14b2d6e5482 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 18:00:05 -0400 Subject: [PATCH 11/36] Fix output pillar for minio --- .../logstash/pipelines/config/so/9998_output_minio.conf.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja index a085ee587..060f42daf 100644 --- a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja +++ b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja @@ -1,6 +1,6 @@ {%- set MANAGER = salt['pillar.get']('global:managerip', '') -%} -{%- set access_key = salt['pillar.get']('global:access_key', '') %} -{%- set access_secret = salt['pillar.get']('global:access_secret', '') %} +{%- set access_key = salt['pillar.get']('minio:access_key', '') %} +{%- set access_secret = salt['pillar.get']('minio:access_secret', '') %} output { s3 { access_key_id => "{{ access_key }}" From a2e5dca06529bc3f4fb1ea938e2f6a50f605acc9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 18:02:54 -0400 Subject: [PATCH 12/36] Fix output pillar for minio --- salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja index 1f6bf03b4..33a5e9055 100644 --- a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja +++ b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja @@ -4,8 +4,8 @@ {%- set MANAGER = salt['pillar.get']('global:managerip', '') %} {% endif -%} {%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} -{%- set access_key = salt['pillar.get']('global:access_key', '') %} -{%- set access_secret = salt['pillar.get']('global:access_secret', '') %} +{%- set access_key = salt['pillar.get']('minio:access_key', '') %} +{%- set access_secret = salt['pillar.get']('minio:access_secret', '') %} input { s3 { access_key_id => "{{ access_key }}" From 61ff944087edd51d51ef29305713ed03c71c1b0e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 18:18:06 -0400 Subject: [PATCH 13/36] add tmp to survive restarts --- salt/logstash/init.sls | 2 +- salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja | 1 + setup/so-setup | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index b63c1ce96..85590673d 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -127,7 +127,7 @@ importdir: # Create the logstash data directory nsmlsdir: file.directory: - - name: /nsm/logstash + - name: /nsm/logstash/tmp - user: 931 - group: 939 - makedirs: True diff --git a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja index 060f42daf..0d8efa4c4 100644 --- a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja +++ b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja @@ -10,6 +10,7 @@ output { size_file => 2048 time_file => 1 codec => json + temporary_directory => "/usr/share/logstash/data/tmp" additional_settings => { "force_path_style" => true } diff --git a/setup/so-setup b/setup/so-setup index 7335b5acc..7f127fc57 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -429,6 +429,7 @@ fi update_sudoers >> $setup_log 2>&1 set_progress_str 12 'Generating manager global pillar' + minio_generate_keys manager_global >> $setup_log 2>&1 set_progress_str 13 'Generating manager pillar' From 5d4a0c53b580bc56ca55720785b29ebfaae130f2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 21:29:07 -0400 Subject: [PATCH 14/36] add ssl cert for minio --- salt/ssl/init.sls | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 1cef1bf0a..d7c84675e 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -181,6 +181,41 @@ regkeyperms: - mode: 640 - group: 939 +/etc/pki/minio.key: + x509.private_key_managed: + - CN: {{ manager }} + - bits: 4096 + - days_remaining: 0 + - days_valid: 820 + - backup: True + - new: True + {% if salt['file.file_exists']('/etc/pki/minio.key') -%} + - prereq: + - x509: /etc/pki/minio.crt + {%- endif %} + +# Create a cert for the docker registry +/etc/pki/minio.crt: + x509.certificate_managed: + - ca_server: {{ ca_server }} + - signing_policy: registry + - public_key: /etc/pki/minio.key + - CN: {{ manager }} + - days_remaining: 0 + - days_valid: 820 + - backup: True + - unless: + # https://github.com/saltstack/salt/issues/52167 + # Will trigger 5 days (432000 sec) from cert expiration + - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/minio.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' + +miniokeyperms: + file.managed: + - replace: False + - name: /etc/pki/minio.key + - mode: 640 + - group: 939 + /etc/pki/managerssl.key: x509.private_key_managed: - CN: {{ manager }} From a733dceb180f6ed8a5c94610bcc1919115bc2cb1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 22:33:40 -0400 Subject: [PATCH 15/36] enable ssl minio --- salt/minio/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/minio/init.sls b/salt/minio/init.sls index d77c775aa..391f7f811 100644 --- a/salt/minio/init.sls +++ b/salt/minio/init.sls @@ -55,4 +55,6 @@ minio: - binds: - /nsm/minio/data:/data:rw - /opt/so/conf/minio/etc:/root/.minio:rw + - /etc/pki/minio.key:/root/.minio/certs/private.key:ro + - /etc/pki/minio.crt:/root/.minio/certs/private.crt:ro - entrypoint: "/usr/bin/docker-entrypoint.sh server --address :9595 /data" From a765790d6c1575de96c337355da0e6965704839d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 22:37:04 -0400 Subject: [PATCH 16/36] fix minio container name --- salt/minio/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/minio/init.sls b/salt/minio/init.sls index 391f7f811..2dca6cca3 100644 --- a/salt/minio/init.sls +++ b/salt/minio/init.sls @@ -42,7 +42,7 @@ logstashbucket: - group: 939 - makedirs: True -minio: +so-minio: docker_container.running: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-minio:{{ VERSION }} - hostname: so-minio From 58872c9b4817ca82cd7f8cd33bc1a62d48406a93 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 22:40:59 -0400 Subject: [PATCH 17/36] enable ssl logstash --- salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja index 33a5e9055..7358cf6e3 100644 --- a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja +++ b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja @@ -10,7 +10,7 @@ input { s3 { access_key_id => "{{ access_key }}" secret_access_key => "{{ access_secret }}" - endpoint => "http://{{ MANAGER }}:9595" + endpoint => "https://{{ MANAGER }}:9595" bucket => "logstash" delete => true interval => 10 diff --git a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja index 0d8efa4c4..4092b6edd 100644 --- a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja +++ b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja @@ -5,7 +5,7 @@ output { s3 { access_key_id => "{{ access_key }}" secret_access_key => "{{ access_secret}}" - endpoint => "http://{{ MANAGER }}:9595" + endpoint => "https://{{ MANAGER }}:9595" bucket => "logstash" size_file => 2048 time_file => 1 From 970ee195a1e274de7b74cbe0c44a9736a6e0c527 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 23:08:33 -0400 Subject: [PATCH 18/36] use hostname so TLS will work --- salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja | 4 ++-- .../logstash/pipelines/config/so/9998_output_minio.conf.jinja | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja index 7358cf6e3..27b287532 100644 --- a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja +++ b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja @@ -1,7 +1,7 @@ {%- if grains.role == 'so-heavynode' %} -{%- set MANAGER = salt['pillar.get']('elasticsearch:mainip', '') %} +{%- set MANAGER = salt['grains.get']('host') %} {%- else %} -{%- set MANAGER = salt['pillar.get']('global:managerip', '') %} +{%- set MANAGER = salt['grains.get']('master') %} {% endif -%} {%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} {%- set access_key = salt['pillar.get']('minio:access_key', '') %} diff --git a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja index 4092b6edd..34a044f34 100644 --- a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja +++ b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja @@ -1,4 +1,4 @@ -{%- set MANAGER = salt['pillar.get']('global:managerip', '') -%} +{%- set MANAGER = salt['grains.get']('master') %} {%- set access_key = salt['pillar.get']('minio:access_key', '') %} {%- set access_secret = salt['pillar.get']('minio:access_secret', '') %} output { From 1855eeaa139102f8b820ec0f3b43dbd1f594aadb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 23:09:08 -0400 Subject: [PATCH 19/36] fix cert name --- salt/minio/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/minio/init.sls b/salt/minio/init.sls index 2dca6cca3..44c89d4d4 100644 --- a/salt/minio/init.sls +++ b/salt/minio/init.sls @@ -56,5 +56,5 @@ so-minio: - /nsm/minio/data:/data:rw - /opt/so/conf/minio/etc:/root/.minio:rw - /etc/pki/minio.key:/root/.minio/certs/private.key:ro - - /etc/pki/minio.crt:/root/.minio/certs/private.crt:ro + - /etc/pki/minio.crt:/root/.minio/certs/public.crt:ro - entrypoint: "/usr/bin/docker-entrypoint.sh server --address :9595 /data" From 734f2979d27b283a7cf2bf243859275fa68dc405 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Aug 2020 23:20:51 -0400 Subject: [PATCH 20/36] add ca.crt to lgostash docker bind --- salt/logstash/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 85590673d..ffaee296b 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -166,6 +166,7 @@ so-logstash: - /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro + - /etc/pki/ca.crt:/etc/ssl/certs/ca.crt:ro {%- if grains['role'] == 'so-eval' %} - /nsm/zeek:/nsm/zeek:ro - /nsm/suricata:/suricata:ro From e30746c5ca2d43396e0d8f78556d63ca205a4c4c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Aug 2020 14:12:06 -0400 Subject: [PATCH 21/36] Final minio fix --- salt/minio/init.sls | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/salt/minio/init.sls b/salt/minio/init.sls index 44c89d4d4..ece8673bd 100644 --- a/salt/minio/init.sls +++ b/salt/minio/init.sls @@ -19,11 +19,10 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} - # Minio Setup minioconfdir: file.directory: - - name: /opt/so/conf/minio/etc + - name: /opt/so/conf/minio/etc/certs - user: 939 - group: 939 - makedirs: True @@ -54,7 +53,7 @@ so-minio: - MINIO_SECRET_KEY: {{ access_secret }} - binds: - /nsm/minio/data:/data:rw - - /opt/so/conf/minio/etc:/root/.minio:rw - - /etc/pki/minio.key:/root/.minio/certs/private.key:ro - - /etc/pki/minio.crt:/root/.minio/certs/public.crt:ro - - entrypoint: "/usr/bin/docker-entrypoint.sh server --address :9595 /data" + - /opt/so/conf/minio/etc:/.minio:rw + - /etc/pki/minio.key:/.minio/certs/private.key:ro + - /etc/pki/minio.crt:/.minio/certs/public.crt:ro + - entrypoint: "/usr/bin/docker-entrypoint.sh server --certs-dir /.minio/certs --address :9595 /data" \ No newline at end of file From 95cae2f17ac534247cc92cced3952f79a260df0a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Aug 2020 14:14:35 -0400 Subject: [PATCH 22/36] SSL path for logstash --- salt/logstash/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index ffaee296b..356a3aceb 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -148,6 +148,7 @@ so-logstash: - user: logstash - environment: - LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }} + - SSL_CERT_FILE=/etc/ssl/certs/ca.crt - port_bindings: {% for BINDING in DOCKER_OPTIONS.port_bindings %} - {{ BINDING }} From 64c366971fc54fccd84854da82cccbf9462a5f46 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 5 Aug 2020 16:13:25 -0400 Subject: [PATCH 23/36] [fix] Redirect ca state apply in setup to /dev/null Redirect ca state apply line in accept_salt_key_remote to /dev/null to avoid generating error in setup log --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index ad4b4252f..837df5eb5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -27,7 +27,7 @@ accept_salt_key_remote() { echo "Accept the key remotely on the manager" >> "$setup_log" 2>&1 # Delete the key just in case. ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo salt-key -d "$MINION_ID" -y - salt-call state.apply ca + salt-call state.apply ca >> /dev/null 2>&1 ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo salt-key -a "$MINION_ID" -y } From 30ff6d2b93def2d8cb2640787c3561f5e4e375be Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 5 Aug 2020 16:28:32 -0400 Subject: [PATCH 24/36] Update event fields to reflect new ECS terms - WIP --- salt/soc/files/soc/soc.json | 70 ++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 31e49fc86..d64f95983 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -33,44 +33,44 @@ "mostRecentlyUsedLimit": 5, "eventFields": { "default": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid", "network.community_id", "event.dataset" ], - "bro_conn": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "protocol", "service", "log.id.uid" ], - "bro_dce_rpc": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "endpoint", "named_pipe", "operation", "log.id.uid" ], - "bro_dhcp": ["soc_timestamp", "source.ip", "destination.ip", "domain_name", "hostname", "message_types", "log.id.uid" ], - "bro_dnp3": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "fc_reply", "log.id.uid" ], - "bro_dns": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "protocol", "query", "query_type_name", "rcode_name", "log.id.uid" ], - "bro_dpd": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid" ], - "bro_files": ["soc_timestamp", "source.ip", "destination.ip", "log.id.flog.id.uid", "mimetype", "source", "log.id.uid" ], - "bro_ftp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ftp_argument", "ftp_command", "reply_code", "log.id.uid", "username" ], - "bro_http": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "method", "virtual_host", "status_code", "status_message", "log.id.uid" ], - "bro_intel": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "indicator", "indicator_type", "seen_where", "log.id.uid" ], - "bro_irc": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "irc_command", "log.id.uid", "value" ], - "bro_kerberos": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "client", "service", "request_type", "log.id.uid" ], - "bro_modbus": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "function", "log.id.uid" ], - "bro_mysql": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "mysql_argument", "mysql_command", "mysql_success", "response", "log.id.uid" ], - "bro_notice": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "note", "msg", "log.id.uid" ], - "bro_ntlm": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "hostname", "ntlm_success", "server_dns_computer_name", "server_nb_computer_name", "server_tree_name", "log.id.uid" ], - "bro_pe": ["soc_timestamp", "is_64bit", "is_exe", "machine", "os", "subsystem", "log.id.flog.id.uid" ], - "bro_radius": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid", "username", "framed_addr", "reply_msg", "result" ], - "bro_rdp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "client_build", "client_name", "cookie", "encryption_level", "encryption_method", "keyboard_layout", "result", "security_protocol", "log.id.uid" ], - "bro_rfb": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "authentication_method", "auth", "share_flag", "desktop_name", "log.id.uid" ], - "bro_signatures" : ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "note", "signature_id", "event_message", "sub_message", "signature_count", "host_count", "log.id.uid" ], - "bro_sip": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "method", "uri", "request_from", "request_to", "response_from", "response_to", "call_id", "subject", "user_agent", "status_code", "log.id.uid" ], - "bro_smb_files" : ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.flog.id.uid", "action", "path", "name", "size", "prev_name", "log.id.uid" ], - "bro_smb_mapping" : ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "path", "service", "share_type", "log.id.uid" ], - "bro_smtp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "helo", "mail_from", "recipient_to", "from", "to", "cc", "reply_to", "subject", "useragent", "log.id.uid" ], - "bro_snmp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "community", "version", "log.id.uid" ], - "bro_socks": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid" ], - "bro_software": ["soc_timestamp", "source.ip", "name", "software_type" ], - "bro_ssh": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "version", "hassh", "direction", "client", "server", "log.id.uid" ], - "bro_ssl": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "cipher", "curve", "server_name", "log.id.uid", "validation_status", "version" ], - "bro_syslog": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "facility", "protocol", "severity", "syslog-priority", "log.id.uid" ], - "bro_tunnels": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "tunnel_type", "action", "log.id.uid" ], - "bro_weird": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "name", "log.id.uid" ], - "bro_x509": ["soc_timestamp", "certificate_common_name", "certificate_country_code", "certificate_key_length", "issuer_organization", "log.id.id" ], + "zeek:conn": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "network.transport", "network.protocol", "log.id.uid" ], + "zeek:dce_rpc": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "endpoint", "named_pipe", "operation", "log.id.uid" ], + "zeek:dhcp": ["soc_timestamp", "source.ip", "destination.ip", "domain_name", "hostname", "message_types", "log.id.uid" ], + "zeek:dnp3": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "fc_reply", "log.id.uid" ], + "zeek:dns": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "network.transport", "query", "query_type_name", "rcode_name", "log.id.uid" ], + "zeek:dpd": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid" ], + "zeek:files": ["soc_timestamp", "source.ip", "destination.ip", "log.id.flog.id.uid", "mimetype", "source", "log.id.uid" ], + "zeek:ftp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ftp_argument", "ftp_command", "reply_code", "log.id.uid", "username" ], + "zeek:http": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "method", "virtual_host", "status_code", "status_message", "log.id.uid" ], + "zeek:intel": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "indicator", "indicator_type", "seen_where", "log.id.uid" ], + "zeek:irc": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "irc_command", "log.id.uid", "value" ], + "zeek:kerberos": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "client", "network.protocol", "request_type", "log.id.uid" ], + "zeek:modbus": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "function", "log.id.uid" ], + "zeek:mysql": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "mysql_argument", "mysql_command", "mysql_success", "response", "log.id.uid" ], + "zeek:notice": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "note", "msg", "log.id.uid" ], + "zeek:ntlm": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "hostname", "ntlm_success", "server_dns_computer_name", "server_nb_computer_name", "server_tree_name", "log.id.uid" ], + "zeek:pe": ["soc_timestamp", "is_64bit", "is_exe", "machine", "os", "subsystem", "log.id.flog.id.uid" ], + "zeek:radius": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid", "username", "framed_addr", "reply_msg", "result" ], + "zeek:rdp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "client_build", "client_name", "cookie", "encryption_level", "encryption_method", "keyboard_layout", "result", "security_protocol", "log.id.uid" ], + "zeek:rfb": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "authentication_method", "auth", "share_flag", "desktop_name", "log.id.uid" ], + "zeek:signatures" : ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "note", "signature_id", "event_message", "sub_message", "signature_count", "host_count", "log.id.uid" ], + "zeek:sip": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "method", "uri", "request_from", "request_to", "response_from", "response_to", "call_id", "subject", "user_agent", "status_code", "log.id.uid" ], + "zeek:smb_files" : ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.fuid", "action", "path", "name", "size", "prev_name", "log.id.uid" ], + "zeek:smb_mapping" : ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "path", "network.protocol", "share_type", "log.id.uid" ], + "zeek:smtp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "helo", "mail_from", "recipient_to", "from", "to", "cc", "reply_to", "subject", "useragent", "log.id.uid" ], + "zeek:snmp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "community", "version", "log.id.uid" ], + "zeek:socks": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid" ], + "zeek:software": ["soc_timestamp", "source.ip", "name", "software_type" ], + "zeek:ssh": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ssh.version", "ssh.hassh_version", "ssh.direction", "ssh.client", "ssh.server", "log.id.uid" ], + "zeek:ssl": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ssl.cipher", "ssl.curve", "ssl.certificate.subject", "ssl.validation_status", "ssl.version", "log.id.uid" ], + "zeek:syslog": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "syslog.facility", "network.protocol", "syslog.severity", "log.id.uid" ], + "zeek:tunnels": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "tunnel_type", "action", "log.id.uid" ], + "zeek:weird": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "weird.name", "log.id.uid" ], + "zeek:x509": ["soc_timestamp", "x509.certificate.subject", "x509.certificate.key.type", "x509.certificate.key.length", "x509.certificate.issuer", "log.id.id" ], "cron" : ["soc_timestamp", "message" ], "anacron": ["soc_timestamp", "message" ], "bluetoothd": ["soc_timestamp", "message" ], - "firewall": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "protocol", "direction", "interface", "action", "reason" ], + "firewall": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "network.transport", "direction", "interface", "action", "reason" ], "ntpd" : ["soc_timestamp", "message" ], "ossec": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "alert_level", "classification", "description", "username", "escalated_user", "location", "process" ], "pulseaudio": ["soc_timestamp", "message" ], From 633c100ace35906f3245e67c28c239cf86e84bc5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Aug 2020 16:40:21 -0400 Subject: [PATCH 25/36] final logstash tweaks --- salt/logstash/init.sls | 2 +- salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 356a3aceb..1fa5b0e86 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -167,7 +167,7 @@ so-logstash: - /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro - - /etc/pki/ca.crt:/etc/ssl/certs/ca.crt:ro + - /etc/ssl/certs/intca.crt:/etc/ssl/certs/ca.crt:ro {%- if grains['role'] == 'so-eval' %} - /nsm/zeek:/nsm/zeek:ro - /nsm/suricata:/suricata:ro diff --git a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja index 27b287532..36a81b537 100644 --- a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja +++ b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja @@ -13,7 +13,7 @@ input { endpoint => "https://{{ MANAGER }}:9595" bucket => "logstash" delete => true - interval => 10 + interval => 5 codec => json additional_settings => { "force_path_style" => true From d9b1127308826706a184b8331d56a0aa6e92199c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Aug 2020 22:36:23 -0400 Subject: [PATCH 26/36] Switch to gzip encoding --- salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja index 34a044f34..08c81cee9 100644 --- a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja +++ b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja @@ -10,6 +10,7 @@ output { size_file => 2048 time_file => 1 codec => json + encoding => gzip temporary_directory => "/usr/share/logstash/data/tmp" additional_settings => { "force_path_style" => true From 4e40615e51bea1427f2dfb4cff1d26af56988b3a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Aug 2020 22:47:12 -0400 Subject: [PATCH 27/36] Add tuneable to the global pillar --- .../pipelines/config/so/0899_input_minio.conf.jinja | 3 ++- .../pipelines/config/so/9998_output_minio.conf.jinja | 9 ++++++--- setup/so-functions | 7 ++++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja index 36a81b537..59e457115 100644 --- a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja +++ b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja @@ -6,6 +6,7 @@ {%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} {%- set access_key = salt['pillar.get']('minio:access_key', '') %} {%- set access_secret = salt['pillar.get']('minio:access_secret', '') %} +{%- set INTERVAL = salt['pillar.get']('s3_settings:interval', 5) %} input { s3 { access_key_id => "{{ access_key }}" @@ -13,7 +14,7 @@ input { endpoint => "https://{{ MANAGER }}:9595" bucket => "logstash" delete => true - interval => 5 + interval => {{ INTERVAL }} codec => json additional_settings => { "force_path_style" => true diff --git a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja index 08c81cee9..37f829ec0 100644 --- a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja +++ b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja @@ -1,16 +1,19 @@ {%- set MANAGER = salt['grains.get']('master') %} {%- set access_key = salt['pillar.get']('minio:access_key', '') %} {%- set access_secret = salt['pillar.get']('minio:access_secret', '') %} +{%- set SIZE_FILE = salt['pillar.get']('s3_settings:size_file', 2048) %} +{%- set TIME_FILE = salt['pillar.get']('s3_settings:time_file', 1) %} +{%- set ENCODING = salt['pillar.get']('s3_settings:encoding', 'gzip') %} output { s3 { access_key_id => "{{ access_key }}" secret_access_key => "{{ access_secret}}" endpoint => "https://{{ MANAGER }}:9595" bucket => "logstash" - size_file => 2048 - time_file => 1 + size_file => {{ SIZE_FILE }} + time_file => {{ TIME_FILE }} codec => json - encoding => gzip + encoding => {{ ENCODING }} temporary_directory => "/usr/share/logstash/data/tmp" additional_settings => { "force_path_style" => true diff --git a/setup/so-functions b/setup/so-functions index de14447e4..fdf667d76 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1121,7 +1121,12 @@ manager_global() { " delete: 45"\ "minio:"\ " access_key: $ACCESS_KEY"\ - " access_secret: $ACCESS_SECRET" > "$global_pillar" + " access_secret: $ACCESS_SECRET"\ + "s3_settings:"\ + " size_file: 2048"\ + " time_file: 1"\ + " encoding: gzip"\ + " interval: 5" > "$global_pillar" printf '%s\n' '----' >> "$setup_log" 2>&1 cat "$global_pillar" >> "$setup_log" 2>&1 From e7225349a6133c925270c994b8735acc2d678c06 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Aug 2020 22:56:41 -0400 Subject: [PATCH 28/36] Ability to toggle between redis and minio --- pillar/logstash/manager.sls | 5 +++++ pillar/logstash/search.sls | 5 +++++ setup/so-functions | 1 + 3 files changed, 11 insertions(+) diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls index 861b8f665..dcf222ae4 100644 --- a/pillar/logstash/manager.sls +++ b/pillar/logstash/manager.sls @@ -1,7 +1,12 @@ +{%- set PIPELINE = salt['pillar.get']('global:pipeline', 'minio') %} logstash: pipelines: manager: config: - so/0009_input_beats.conf - so/0010_input_hhbeats.conf + {%- if PIPELINE == "minio"%} - so/9998_output_minio.conf.jinja + {%- else %} + - so/9999_output_redis.conf.jinja + {%- endif %} \ No newline at end of file diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index cad849153..22f73c5d4 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -1,8 +1,13 @@ +{%- set PIPELINE = salt['pillar.get']('global:pipeline', 'minio') %} logstash: pipelines: search: config: + {%- if PIPELINE == "minio"%} - so/0899_input_minio.conf.jinja + {%- else %} + - so/0900_input_redis.conf.jinja + {%- endif %} - so/9000_output_zeek.conf.jinja - so/9002_output_import.conf.jinja - so/9034_output_syslog.conf.jinja diff --git a/setup/so-functions b/setup/so-functions index fdf667d76..d965a8b86 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1049,6 +1049,7 @@ manager_global() { " wazuh: $WAZUH"\ " managerupdate: $MANAGERUPDATES"\ " imagerepo: $IMAGEREPO"\ + " pipeline: minio"\ "pcap:"\ " sensor_checkin_interval_ms: $SENSOR_CHECKIN_INTERVAL_MS"\ "strelka:"\ From 4f9ef890980eee18b2184902bdf34f77385b9d71 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 6 Aug 2020 14:30:44 -0400 Subject: [PATCH 29/36] Simplify elastalert rules --- salt/elastalert/files/rules/so/suricata_thehive.yaml | 8 ++------ salt/elastalert/files/rules/so/wazuh_thehive.yaml | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/salt/elastalert/files/rules/so/suricata_thehive.yaml b/salt/elastalert/files/rules/so/suricata_thehive.yaml index 0135edadd..8657d4168 100644 --- a/salt/elastalert/files/rules/so/suricata_thehive.yaml +++ b/salt/elastalert/files/rules/so/suricata_thehive.yaml @@ -8,14 +8,10 @@ es_host: {{es}} es_port: 9200 name: Suricata-Alert -type: frequency +type: any index: "*:so-ids-*" -num_events: 1 -timeframe: - minutes: 10 buffer_time: - minutes: 10 -allow_buffer_time_overlap: true + minutes: 5 query_key: ["rule.uuid","source.ip","destination.ip"] realert: days: 1 diff --git a/salt/elastalert/files/rules/so/wazuh_thehive.yaml b/salt/elastalert/files/rules/so/wazuh_thehive.yaml index 8aa085566..7fd49e23e 100644 --- a/salt/elastalert/files/rules/so/wazuh_thehive.yaml +++ b/salt/elastalert/files/rules/so/wazuh_thehive.yaml @@ -8,14 +8,10 @@ es_host: {{es}} es_port: 9200 name: Wazuh-Alert -type: frequency +type: any index: "*:so-ossec-*" -num_events: 1 -timeframe: - minutes: 10 buffer_time: - minutes: 10 -allow_buffer_time_overlap: true + minutes: 5 realert: days: 1 filter: From 31fd0b6407a4cc70bd4cbfe2848b30cf2fe9a5cb Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 6 Aug 2020 14:59:32 -0400 Subject: [PATCH 30/36] Update the Hunt event fields lookups to reflect the latest ingest configs --- salt/soc/files/soc/soc.json | 88 +++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 47 deletions(-) diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index b098931ba..999819356 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -33,53 +33,47 @@ "mostRecentlyUsedLimit": 5, "eventFields": { "default": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid", "network.community_id", "event.dataset" ], - "zeek:conn": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "network.transport", "network.protocol", "log.id.uid" ], - "zeek:dce_rpc": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "endpoint", "named_pipe", "operation", "log.id.uid" ], - "zeek:dhcp": ["soc_timestamp", "source.ip", "destination.ip", "domain_name", "hostname", "message_types", "log.id.uid" ], - "zeek:dnp3": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "fc_reply", "log.id.uid" ], - "zeek:dns": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "network.transport", "query", "query_type_name", "rcode_name", "log.id.uid" ], - "zeek:dpd": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid" ], - "zeek:files": ["soc_timestamp", "source.ip", "destination.ip", "log.id.flog.id.uid", "mimetype", "source", "log.id.uid" ], - "zeek:ftp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ftp_argument", "ftp_command", "reply_code", "log.id.uid", "username" ], - "zeek:http": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "method", "virtual_host", "status_code", "status_message", "log.id.uid" ], - "zeek:intel": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "indicator", "indicator_type", "seen_where", "log.id.uid" ], - "zeek:irc": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "irc_command", "log.id.uid", "value" ], - "zeek:kerberos": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "client", "network.protocol", "request_type", "log.id.uid" ], - "zeek:modbus": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "function", "log.id.uid" ], - "zeek:mysql": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "mysql_argument", "mysql_command", "mysql_success", "response", "log.id.uid" ], - "zeek:notice": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "note", "msg", "log.id.uid" ], - "zeek:ntlm": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "hostname", "ntlm_success", "server_dns_computer_name", "server_nb_computer_name", "server_tree_name", "log.id.uid" ], - "zeek:pe": ["soc_timestamp", "is_64bit", "is_exe", "machine", "os", "subsystem", "log.id.flog.id.uid" ], - "zeek:radius": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid", "username", "framed_addr", "reply_msg", "result" ], - "zeek:rdp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "client_build", "client_name", "cookie", "encryption_level", "encryption_method", "keyboard_layout", "result", "security_protocol", "log.id.uid" ], - "zeek:rfb": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "authentication_method", "auth", "share_flag", "desktop_name", "log.id.uid" ], - "zeek:signatures" : ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "note", "signature_id", "event_message", "sub_message", "signature_count", "host_count", "log.id.uid" ], - "zeek:sip": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "method", "uri", "request_from", "request_to", "response_from", "response_to", "call_id", "subject", "user_agent", "status_code", "log.id.uid" ], - "zeek:smb_files" : ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.fuid", "action", "path", "name", "size", "prev_name", "log.id.uid" ], - "zeek:smb_mapping" : ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "path", "network.protocol", "share_type", "log.id.uid" ], - "zeek:smtp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "helo", "mail_from", "recipient_to", "from", "to", "cc", "reply_to", "subject", "useragent", "log.id.uid" ], - "zeek:snmp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "community", "version", "log.id.uid" ], - "zeek:socks": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid" ], - "zeek:software": ["soc_timestamp", "source.ip", "name", "software_type" ], - "zeek:ssh": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ssh.version", "ssh.hassh_version", "ssh.direction", "ssh.client", "ssh.server", "log.id.uid" ], - "zeek:ssl": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ssl.cipher", "ssl.curve", "ssl.certificate.subject", "ssl.validation_status", "ssl.version", "log.id.uid" ], - "zeek:syslog": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "syslog.facility", "network.protocol", "syslog.severity", "log.id.uid" ], - "zeek:tunnels": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "tunnel_type", "action", "log.id.uid" ], - "zeek:weird": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "weird.name", "log.id.uid" ], - "zeek:x509": ["soc_timestamp", "x509.certificate.subject", "x509.certificate.key.type", "x509.certificate.key.length", "x509.certificate.issuer", "log.id.id" ], - "cron" : ["soc_timestamp", "message" ], - "anacron": ["soc_timestamp", "message" ], - "bluetoothd": ["soc_timestamp", "message" ], - "firewall": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "network.transport", "direction", "interface", "action", "reason" ], - "ntpd" : ["soc_timestamp", "message" ], - "ossec": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "alert_level", "classification", "description", "username", "escalated_user", "location", "process" ], - "pulseaudio": ["soc_timestamp", "message" ], - "snort": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "sid", "alert", "category", "classification", "severity" ], - "su" : ["soc_timestamp", "message" ], - "sudo" : ["soc_timestamp", "message" ], - "systemd": ["soc_timestamp", "message" ], - "sysmon": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "host.name", "event.dataset", "parent_image_path", "source_name", "task", "user.name" ], - "wineventlog": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "host.name", "event.code", "event.dataset", "source_name", "task" ] + "::conn": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "network.transport", "network.protocol", "log.id.uid" ], + "::dce_rpc": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "dce_rpc.endpoint", "dce_rpc.named_pipe", "dce_rpc.operation", "log.id.uid" ], + "::dhcp": ["soc_timestamp", "source.ip", "destination.ip", "host.domain", "host.hostname", "dhcp.message_types", "log.id.uid" ], + "::dnp3": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "dnp3.fc_reply", "log.id.uid" ], + "::dns": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "network.transport", "dns.query.name", "dns.query.type_name", "dns.response.code_name", "log.id.uid" ], + "::dpd": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "network.protocol", "observer.analyser", "error.reason", "log.id.uid" ], + "::files": ["soc_timestamp", "source.ip", "destination.ip", "file.name", "file.mime_type", "file.source", "file.bytes.total", "log.id.fuid", "log.id.uid" ], + "::ftp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ftp.user", "ftp.command", "ftp.argument", "ftp.reply_code", "file.size", "log.id.uid" ], + "::http": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "http.method", "http.virtual_host", "http.status_code", "http.status_message", "http.request.body.length", "http.response.body.length", "log.id.uid" ], + "::intel": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "intel.indicator", "intel.indicator_type", "intel.seen_where", "log.id.uid" ], + "::irc": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "irc.username", "irc.nickname", "irc.command.type", "irc.command.value", "irc.command.info", "log.id.uid" ], + "::kerberos": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "kerberos.client", "kerberos.service", "kerberos.request_type", "log.id.uid" ], + "::modbus": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "modbus.function", "log.id.uid" ], + "::mysql": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "mysql.command", "mysql.argument", "mysql.success", "mysql.response", "log.id.uid" ], + "::notice": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "notice.note", "notice.message", "log.id.fuid", "log.id.uid" ], + "::ntlm": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ntlm.name", "ntlm.success", "ntlm.server.dns.name", "ntlm.server.nb.name", "ntlm.server.tree.name", "log.id.uid" ], + "::pe": ["soc_timestamp", "file.is_64bit", "file.is_exe", "file.machine", "file.os", "file.subsystem", "log.id.fuid" ], + "::radius": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid", "username", "radius.framed_address", "radius.reply_message", "radius.result" ], + "::rdp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "rdp.client_build", "client_name", "rdp.cookie", "rdp.encryption_level", "rdp.encryption_method", "rdp.keyboard_layout", "rdp.result", "rdp.security_protocol", "log.id.uid" ], + "::rfb": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "rfb.authentication.method", "rfb.authentication.success", "rfb.share_flag", "rfb.desktop.name", "log.id.uid" ], + "::signatures" : ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "note", "signature_id", "event_message", "sub_message", "signature_count", "host.count", "log.id.uid" ], + "::sip": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "sip.method", "sip.uri", "sip.request.from", "sip.request.to", "sip.response.from", "sip.response.to", "sip.call_id", "sip.subject", "sip.user_agent", "sip.status_code", "log.id.uid" ], + "::smb_files" : ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.fuid", "file.action", "file.path", "file.name", "file.size", "file.prev_name", "log.id.uid" ], + "::smb_mapping" : ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "smb.path", "smb.service", "smb.share_type", "log.id.uid" ], + "::smtp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "smtp.helo", "smtp.mail_from", "smtp.recipient_to", "smtp.from", "smtp.to", "smtp.cc", "smtp.reply_to", "smtp.subject", "smtp.useragent", "log.id.uid" ], + "::snmp": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "snmp.community", "snmp.version", "log.id.uid" ], + "::socks": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "socks.name", "socks.request.host", "socks.request.port", "socks.status", "log.id.uid" ], + "::software": ["soc_timestamp", "source.ip", "software.name", "software.type" ], + "::ssh": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ssh.version", "ssh.hassh_version", "ssh.direction", "ssh.client", "ssh.server", "log.id.uid" ], + "::ssl": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ssl.cipher", "ssl.curve", "ssl.certificate.subject", "ssl.validation_status", "ssl.version", "log.id.uid" ], + "::syslog": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "syslog.facility", "network.protocol", "syslog.severity", "log.id.uid" ], + "::tunnels": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "tunnel_type", "action", "log.id.uid" ], + "::weird": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "weird.name", "log.id.uid" ], + "::x509": ["soc_timestamp", "x509.certificate.subject", "x509.certificate.key.type", "x509.certificate.key.length", "x509.certificate.issuer", "log.id.id" ], + ":firewall:": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "network.transport", "direction", "interface", "action", "reason" ], + ":osquery:": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "source.hostname", "event.dataset", "process.executable", "user.name" ], + ":ossec:": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "rule.name", "rule.level", "rule.category", "process.name", "user.name", "user.escalated", "location", "process.name" ], + ":strelka:file": ["soc_timestamp", "scan.exiftool.OriginalFileName", "file.size", "hash.md5", "scan.exiftool.CompanyName", "scan.exiftool.Description", "scan.exiftool.Directory", "scan.exiftool.FileType", "scan.exiftool.FileOS", "log.id.fuid" ], + ":suricata:": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "rule.gid", "rule.name", "rule.category", "rule.rev", "event.severity", "event.severity_label" ], + ":sysmon:": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "source.hostname", "event.dataset", "process.executable", "user.name" ], + ":windows_eventlog:": ["soc_timestamp", "user.name" ] }, "queries": [ { "name": "Default Query", "description": "Show all events grouped by the origin host", "query": "* | groupby observer.name"}, From 63e31bd6b9e875eb202c393c238b015aa7d18ee7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 6 Aug 2020 15:33:48 -0400 Subject: [PATCH 31/36] Add upload queue thread --- setup/so-functions | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 002ed8d81..b3dc5b060 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -27,7 +27,7 @@ accept_salt_key_remote() { echo "Accept the key remotely on the manager" >> "$setup_log" 2>&1 # Delete the key just in case. ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo salt-key -d "$MINION_ID" -y - salt-call state.apply ca >> /dev/null 2>&1 + salt-call state.apply ca ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo salt-key -a "$MINION_ID" -y } @@ -1126,6 +1126,7 @@ manager_global() { "s3_settings:"\ " size_file: 2048"\ " time_file: 1"\ + " upload_queue_size: 4" " encoding: gzip"\ " interval: 5" > "$global_pillar" From 16d0c02113162aa8244738d58d79722a3bbe5094 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 6 Aug 2020 15:39:02 -0400 Subject: [PATCH 32/36] Fix cert dev null --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index b3dc5b060..d4218a10c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -27,7 +27,7 @@ accept_salt_key_remote() { echo "Accept the key remotely on the manager" >> "$setup_log" 2>&1 # Delete the key just in case. ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo salt-key -d "$MINION_ID" -y - salt-call state.apply ca + salt-call state.apply ca >> /dev/null 2>&1 ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo salt-key -a "$MINION_ID" -y } From bbdaee28ed56cc813f44eec5a91382f025869cea Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 6 Aug 2020 15:41:10 -0400 Subject: [PATCH 33/36] Add upload queue thread --- salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja index 37f829ec0..e953c3521 100644 --- a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja +++ b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja @@ -3,6 +3,7 @@ {%- set access_secret = salt['pillar.get']('minio:access_secret', '') %} {%- set SIZE_FILE = salt['pillar.get']('s3_settings:size_file', 2048) %} {%- set TIME_FILE = salt['pillar.get']('s3_settings:time_file', 1) %} +{%- set UPLOAD_QUEUE_SIZE = salt['pillar.get']('s3_settings:upload_queue_size', 4) %} {%- set ENCODING = salt['pillar.get']('s3_settings:encoding', 'gzip') %} output { s3 { @@ -14,6 +15,7 @@ output { time_file => {{ TIME_FILE }} codec => json encoding => {{ ENCODING }} + upload_queue_size => {{ UPLOAD_QUEUE_SIZE }} temporary_directory => "/usr/share/logstash/data/tmp" additional_settings => { "force_path_style" => true From ddd099233a1111dadd0bba37571162fdb81c9080 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 6 Aug 2020 15:43:45 -0400 Subject: [PATCH 34/36] Playbook Fixes - Issue #1064 --- salt/elastalert/files/elastalert_config.yaml | 4 ++-- salt/elastalert/files/modules/so/playbook-es.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elastalert/files/elastalert_config.yaml b/salt/elastalert/files/elastalert_config.yaml index 7646e8221..ba2b79448 100644 --- a/salt/elastalert/files/elastalert_config.yaml +++ b/salt/elastalert/files/elastalert_config.yaml @@ -21,7 +21,7 @@ run_every: # ElastAlert will buffer results from the most recent # period of time, in case some log sources are not in real time buffer_time: - minutes: 1 + minutes: 5 # The maximum time between queries for ElastAlert to start at the most recently # run query. When ElastAlert starts, for each rule, it will search elastalert_metadata @@ -38,7 +38,7 @@ es_host: {{ esip }} es_port: {{ esport }} # Sets timeout for connecting to and reading from es_host -es_conn_timeout: 60 +es_conn_timeout: 55 # The maximum number of documents that will be downloaded from Elasticsearch in # a single query. The default is 10,000, and if you expect to get near this number, diff --git a/salt/elastalert/files/modules/so/playbook-es.py b/salt/elastalert/files/modules/so/playbook-es.py index c794bdf12..adc03dd29 100644 --- a/salt/elastalert/files/modules/so/playbook-es.py +++ b/salt/elastalert/files/modules/so/playbook-es.py @@ -16,7 +16,7 @@ class PlaybookESAlerter(Alerter): today = strftime("%Y.%m.%d", gmtime()) timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S", gmtime()) headers = {"Content-Type": "application/json"} - payload = {"rule.name": self.rule['play_title'],"event.severity": self.rule['event.severity'],"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"event.module": self.rule['event.module'],"event.dataset": self.rule['event.dataset'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"rule.category": self.rule['rule.category'],"data": match, "@timestamp": timestamp} + payload = {"rule.name": self.rule['play_title'],"event.severity": self.rule['event.severity'],"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"event.module": self.rule['event.module'],"event.dataset": self.rule['event.dataset'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"rule.category": self.rule['rule.category'],"alert_data": match, "@timestamp": timestamp} url = f"http://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/" requests.post(url, data=json.dumps(payload), headers=headers, verify=False) From d3e6657b455b03c0f91821a7623c1effbf4ae170 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 7 Aug 2020 10:01:40 -0400 Subject: [PATCH 35/36] Fix Spacing --- setup/so-functions | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d4218a10c..87b6b5756 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1047,7 +1047,7 @@ manager_global() { " fleet_ip: N/A"\ " sensoronikey: $SENSORONIKEY"\ " wazuh: $WAZUH"\ - " managerupdate: $MANAGERUPDATES"\ + " managerupdate: $MANAGERUPDATES"\ " imagerepo: $IMAGEREPO"\ " pipeline: minio"\ "pcap:"\ @@ -1066,9 +1066,9 @@ manager_global() { " discovery_nodes: 1"\ " hot_warm_enabled: False"\ " cluster_routing_allocation_disk.threshold_enabled: true"\ - " cluster_routing_allocation_disk_watermark_low: 95%"\ - " cluster_routing_allocation_disk_watermark_high: 98%"\ - " cluster_routing_allocation_disk_watermark_flood_stage: 98%"\ + " cluster_routing_allocation_disk_watermark_low: 95%"\ + " cluster_routing_allocation_disk_watermark_high: 98%"\ + " cluster_routing_allocation_disk_watermark_flood_stage: 98%"\ " index_settings:"\ " so-beats:"\ " shards: 1"\ From b534d2b975c43fd96f6717286ebd3093ab28e8f5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 7 Aug 2020 10:05:47 -0400 Subject: [PATCH 36/36] Update so-functions --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 87b6b5756..038a0ba6e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1126,7 +1126,7 @@ manager_global() { "s3_settings:"\ " size_file: 2048"\ " time_file: 1"\ - " upload_queue_size: 4" + " upload_queue_size: 4"\ " encoding: gzip"\ " interval: 5" > "$global_pillar"