From 5f4ec414979fc9e8a2616263331051484b6303a9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 14 May 2020 15:05:37 -0400 Subject: [PATCH 01/62] rename PROD to STANDALONE --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index fe9c4c7b1..4ef32bf5b 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -391,7 +391,7 @@ whiptail_install_type() { "SEARCHNODE" "Add a Search Node with parsing" OFF \ "MASTER" "Start a new grid" OFF \ "EVAL" "Evaluate all the things" OFF \ - "PROD" "Standalone full install of everything" OFF \ + "STANDALONE" "Standalone full install of everything" OFF \ "MASTERSEARCH" "Master + Search Node" OFF \ "HEAVYNODE" "Sensor + Search Node" OFF \ "HELIXSENSOR" "Connect this sensor to FireEye Helix" OFF \ From 3064af753b093d1c0728ee7430de6ed619bd93cc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 14 May 2020 15:23:53 -0400 Subject: [PATCH 02/62] allow STANDALONE mode to install salt-master - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/140 --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index aa28d2e8a..e817d4c7b 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1022,7 +1022,7 @@ saltify() { set_progress_str 6 'Installing various dependencies' yum -y install wget nmap-ncat >> "$setup_log" 2>&1 case "$install_type" in - 'MASTER' | 'EVAL' | 'MASTERSEARCH' | 'FLEET' | 'HELIXSENSOR') + 'MASTER' | 'EVAL' | 'MASTERSEARCH' | 'FLEET' | 'HELIXSENSOR' | 'STANDALONE') reserve_group_ids >> "$setup_log" 2>&1 yum -y install epel-release >> "$setup_log" 2>&1 yum -y install sqlite argon2 curl mariadb-devel >> "$setup_log" 2>&1 From 692f04727f531da88cd942123981e75918d5a074 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 14 May 2020 16:06:19 -0400 Subject: [PATCH 03/62] add standalone mode to pillar and state top - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/140 --- pillar/healthcheck/standalone.sls | 5 +++ pillar/top.sls | 12 +++++++ salt/top.sls | 56 +++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 pillar/healthcheck/standalone.sls diff --git a/pillar/healthcheck/standalone.sls b/pillar/healthcheck/standalone.sls new file mode 100644 index 000000000..dd1a027e9 --- /dev/null +++ b/pillar/healthcheck/standalone.sls @@ -0,0 +1,5 @@ +healthcheck: + enabled: False + schedule: 300 + checks: + - zeek diff --git a/pillar/top.sls b/pillar/top.sls index cc6863e22..241bdb90c 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -40,6 +40,18 @@ base: - healthcheck.eval - minions.{{ grains.id }} + '*_standalone': + - logstash + - logstash.master + - logstash.search + - firewall.* + - data.* + - brologs + - secrets + - healthcheck.standalone + - static + - minions.{{ grains.id }} + '*_node': - static - firewall.* diff --git a/salt/top.sls b/salt/top.sls index 68e05959c..2874be4d6 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -156,6 +156,62 @@ base: - domainstats {%- endif %} + '*_standalone': + - ca + - ssl + - registry + - master + - common + - nginx + - telegraf + - influxdb + - grafana + - soc + - firewall + - idstools + - healthcheck + {%- if FLEETMASTER or FLEETNODE or PLAYBOOK != 0 %} + - mysql + {%- endif %} + {%- if WAZUH != 0 %} + - wazuh + {%- endif %} + - elasticsearch + - logstash + - kibana + - pcap + - suricata + - zeek + {%- if STRELKA %} + - strelka + {%- endif %} + - filebeat + - curator + - elastalert + {%- if FLEETMASTER or FLEETNODE %} + - fleet + - redis + - fleet.install_package + {%- endif %} + - utility + - schedule + - soctopus + {%- if THEHIVE != 0 %} + - hive + {%- endif %} + {%- if PLAYBOOK != 0 %} + - playbook + {%- endif %} + {%- if NAVIGATOR != 0 %} + - navigator + {%- endif %} + {%- if FREQSERVER != 0 %} + - freqserver + {%- endif %} + {%- if DOMAINSTATS != 0 %} + - domainstats + {%- endif %} + # Search node logic '*_node and I@node:node_type:parser': From fb020f4fcdccd07f8dbe355b2b4b6571d549c611 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 14 May 2020 16:26:04 -0400 Subject: [PATCH 04/62] update so-functions to handle STANDALONE mode - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/140 --- setup/so-functions | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index e817d4c7b..369da5973 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -408,7 +408,7 @@ copy_master_config() { copy_minion_tmp_files() { case "$install_type" in - 'MASTER' | 'EVAL' | 'HELIXSENSOR' | 'MASTERSEARCH') + 'MASTER' | 'EVAL' | 'HELIXSENSOR' | 'MASTERSEARCH' | 'STANDALONE') echo "Copying pillar and salt files in $temp_install_dir to /opt/so/saltstack" cp -Rv "$temp_install_dir"/pillar/ /opt/so/saltstack/ >> "$setup_log" 2>&1 if [ -d "$temp_install_dir"/salt ] ; then @@ -767,7 +767,7 @@ got_root() { get_minion_type() { local minion_type case "$install_type" in - 'EVAL' | 'MASTERSEARCH' | 'MASTER' | 'SENSOR' | 'HEAVYNODE' | 'FLEET') + 'EVAL' | 'MASTERSEARCH' | 'MASTER' | 'SENSOR' | 'HEAVYNODE' | 'FLEET' | 'STANDALONE') minion_type=$(echo "$install_type" | tr '[:upper:]' '[:lower:]') ;; 'HELIXSENSOR') @@ -803,7 +803,7 @@ master_pillar() { " freq: 0"\ " domainstats: 0" >> "$pillar_file" - if [ "$install_type" = 'EVAL' ] || [ "$install_type" = 'HELIXSENSOR' ] || [ "$install_type" = 'MASTERSEARCH' ]; then + if [ "$install_type" = 'EVAL' ] || [ "$install_type" = 'HELIXSENSOR' ] || [ "$install_type" = 'MASTERSEARCH' ] || [ "$install_type" = 'STANDALONE' ]; then printf '%s\n'\ " ls_pipeline_batch_size: 125"\ " ls_input_threads: 1"\ @@ -1093,7 +1093,7 @@ saltify() { 'FLEET') if [ "$OSVER" != 'xenial' ]; then apt-get -y install python3-mysqldb >> "$setup_log" 2>&1; else apt-get -y install python-mysqldb >> "$setup_log" 2>&1; fi ;; - 'MASTER' | 'EVAL' | 'MASTERSEARCH') # TODO: should this also be HELIXSENSOR? + 'MASTER' | 'EVAL' | 'MASTERSEARCH' | 'STANDALONE') # TODO: should this also be HELIXSENSOR? if [ "$OSVER" != "xenial" ]; then local py_ver_url_path="/py3"; else local py_ver_url_path="/apt"; fi # Add saltstack repo(s) @@ -1151,7 +1151,7 @@ saltify() { salt_checkin() { case "$install_type" in - 'MASTER' | 'EVAL' | 'HELIXSENSOR' | 'MASTERSEARCH') # Fix Mine usage + 'MASTER' | 'EVAL' | 'HELIXSENSOR' | 'MASTERSEARCH' | 'STANDALONE') # Fix Mine usage { echo "Building Certificate Authority"; salt-call state.apply ca; @@ -1282,7 +1282,7 @@ set_hostname() { set_hostname_iso - if [[ ! $install_type =~ ^(MASTER|EVAL|HELIXSENSOR|MASTERSEARCH)$ ]]; then + if [[ ! $install_type =~ ^(MASTER|EVAL|HELIXSENSOR|MASTERSEARCH|STANDALONE)$ ]]; then if ! getent hosts "$MSRV"; then echo "$MSRVIP $MSRV" >> /etc/hosts fi @@ -1384,7 +1384,7 @@ set_management_interface() { set_node_type() { case "$install_type" in - 'SEARCHNODE' | 'EVAL' | 'MASTERSEARCH' | 'HEAVYNODE') + 'SEARCHNODE' | 'EVAL' | 'MASTERSEARCH' | 'HEAVYNODE' | 'STANDALONE') NODETYPE='search' ;; 'PARSINGNODE') @@ -1450,7 +1450,7 @@ ls_heapsize() { fi case "$install_type" in - 'MASTERSEARCH' | 'HEAVYNODE' | 'HELIXSENSOR') + 'MASTERSEARCH' | 'HEAVYNODE' | 'HELIXSENSOR' | 'STANDALONE') LS_HEAP_SIZE='1000m' ;; 'EVAL') @@ -1462,7 +1462,7 @@ ls_heapsize() { esac export LS_HEAP_SIZE - if [[ "$install_type" =~ ^(EVAL|MASTERSEARCH)$ ]]; then + if [[ "$install_type" =~ ^(EVAL|MASTERSEARCH|STANDALONE)$ ]]; then NODE_LS_HEAP_SIZE=LS_HEAP_SIZE export NODE_LS_HEAP_SIZE fi @@ -1484,7 +1484,7 @@ es_heapsize() { fi export ES_HEAP_SIZE - if [[ "$install_type" =~ ^(EVAL|MASTERSEARCH)$ ]]; then + if [[ "$install_type" =~ ^(EVAL|MASTERSEARCH|STANDALONE)$ ]]; then NODE_ES_HEAP_SIZE=ES_HEAP_SIZE export NODE_ES_HEAP_SIZE fi From e41c1b6755f1de77230a258db74adc19f19c5525 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 14 May 2020 16:57:34 -0400 Subject: [PATCH 05/62] fix typo --- salt/soc/files/soc/soc.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 88eeed8ec..1f7a61eb6 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -88,7 +88,7 @@ { "name": "Alerts", "description": "Show all alerts grouped by alert source", "query": "event.dataset: alert | groupby event.module"}, { "name": "NIDS Alerts", "description": "Show all NIDS alerts grouped by alert name", "query": "event.category: network AND event.dataset: alert | groupby rule.name"}, { "name": "Wazuh/OSSEC Alerts", "description": "Show all Wazuh alerts grouped by category", "query": "event.module:ossec AND event.dataset:alert | groupby rule.category"}, - { "name": "Wazuh/OSSEC Commands", "description": "Show all Wazuh alerts grouped by command line", "query": "eevent.module:ossec AND event.dataset:alert | groupby process.command_line"}, + { "name": "Wazuh/OSSEC Commands", "description": "Show all Wazuh alerts grouped by command line", "query": "event.module:ossec AND event.dataset:alert | groupby process.command_line"}, { "name": "Wazuh/OSSEC Processes", "description": "Show all Wazuh alerts grouped by process name", "query": "event.module:ossec AND event.dataset:alert | groupby process.name"}, { "name": "Wazuh/OSSEC Users", "description": "Show all Wazuh alerts grouped by username", "query": "event.module:ossec AND event.dataset:alert | groupby user.name"}, { "name": "Sysmon Events", "description": "Show all Sysmon logs grouped by event_id", "query": "event_type:sysmon | groupby event_id"}, From 5f1582f39fcf37f2ccf826917e36a4bd3455c9b3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 14 May 2020 16:57:50 -0400 Subject: [PATCH 06/62] remove extra space for standalone from pillar/top = https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/140 --- pillar/top.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pillar/top.sls b/pillar/top.sls index 241bdb90c..ad3b61d90 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -40,7 +40,7 @@ base: - healthcheck.eval - minions.{{ grains.id }} - '*_standalone': + '*_standalone': - logstash - logstash.master - logstash.search From e8244cb2f262eac42481c10fcc92712fd4a63fe5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 14 May 2020 17:14:35 -0400 Subject: [PATCH 07/62] add standalone to case statement in configure_minion so-functions - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/140 --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 369da5973..3f11675f0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -321,7 +321,7 @@ configure_minion() { 'helix') echo "master: $HOSTNAME" >> "$minion_config" ;; - 'master' | 'eval' | 'mastersearch') + 'master' | 'eval' | 'mastersearch' | 'standalone') printf '%s\n'\ "master: $HOSTNAME"\ "mysql.host: '$MAINIP'"\ From 509188092c327c2c9e6e9a1530af64fb2c1fe897 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 15 May 2020 10:02:25 -0400 Subject: [PATCH 08/62] adding so-standalone state logic, add zeek pillar to so-standalone - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/140 --- pillar/top.sls | 2 +- salt/elastalert/init.sls | 21 +++++-------------- salt/elasticsearch/init.sls | 22 +++++++------------- salt/firewall/init.sls | 25 +++++++++++----------- salt/logstash/init.sls | 41 +++++++++++++------------------------ salt/ssl/init.sls | 6 +++--- 6 files changed, 43 insertions(+), 74 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index ad3b61d90..6569209ef 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -2,7 +2,7 @@ base: '*': - patch.needs_restarting - '*_eval or *_helix or *_heavynode or *_sensor': + '*_eval or *_helix or *_heavynode or *_sensor or *_standalone': - match: compound - zeek diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index 08cb0836d..8ee844c91 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -14,24 +14,13 @@ # along with this program. If not, see . {% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} {% set MASTER = salt['grains.get']('master') %} -{% if grains['role'] == 'so-master' %} - -{% set esalert = salt['pillar.get']('master:elastalert', '1') %} -{% set esip = salt['pillar.get']('master:mainip', '') %} -{% set esport = salt['pillar.get']('master:es_port', '') %} - - -{% elif grains['role'] in ['so-eval','so-mastersearch'] %} - -{% set esalert = salt['pillar.get']('master:elastalert', '1') %} -{% set esip = salt['pillar.get']('master:mainip', '') %} -{% set esport = salt['pillar.get']('master:es_port', '') %} - +{% if grains['role'] == ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %} + {% set esalert = salt['pillar.get']('master:elastalert', '1') %} + {% set esip = salt['pillar.get']('master:mainip', '') %} + {% set esport = salt['pillar.get']('master:es_port', '') %} {% elif grains['role'] == 'so-node' %} - -{% set esalert = salt['pillar.get']('node:elastalert', '0') %} - + {% set esalert = salt['pillar.get']('node:elastalert', '0') %} {% endif %} # Elastalert diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 2cdff92c1..d7c017710 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -15,27 +15,19 @@ {% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} {% set MASTER = salt['grains.get']('master') %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} + {% if FEATURES %} {% set FEATURES = "-features" %} {% else %} {% set FEATURES = '' %} {% endif %} -{% if grains['role'] == 'so-master' %} - -{% set esclustername = salt['pillar.get']('master:esclustername', '') %} -{% set esheap = salt['pillar.get']('master:esheap', '') %} - -{% elif grains['role'] in ['so-eval','so-mastersearch'] %} - -{% set esclustername = salt['pillar.get']('master:esclustername', '') %} -{% set esheap = salt['pillar.get']('master:esheap', '') %} - +{% if grains['role'] == in ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %} + {% set esclustername = salt['pillar.get']('master:esclustername', '') %} + {% set esheap = salt['pillar.get']('master:esheap', '') %} {% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} - -{% set esclustername = salt['pillar.get']('node:esclustername', '') %} -{% set esheap = salt['pillar.get']('node:esheap', '') %} - + {% set esclustername = salt['pillar.get']('node:esclustername', '') %} + {% set esheap = salt['pillar.get']('node:esheap', '') %} {% endif %} vm.max_map_count: @@ -149,7 +141,7 @@ so-elasticsearch-pipelines: - file: esyml - file: so-elasticsearch-pipelines-file -{% if grains['role'] == 'so-master' or grains['role'] == "so-eval" or grains['role'] == "so-mastersearch" %} +{% if grains['role'] in ['so-master', 'so-eval', 'so-mastersearch', 'so-standalone'] %} so-elasticsearch-templates: cmd.run: - name: /usr/sbin/so-elasticsearch-templates diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index 85db566f6..43bd6218c 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -1,15 +1,16 @@ # Firewall Magic for the grid -{%- if grains['role'] in ['so-eval','so-master','so-helix','so-mastersearch'] %} -{%- set ip = salt['pillar.get']('static:masterip', '') %} -{%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} -{%- set ip = salt['pillar.get']('node:mainip', '') %} -{%- elif grains['role'] == 'so-sensor' %} -{%- set ip = salt['pillar.get']('sensor:mainip', '') %} -{%- elif grains['role'] == 'so-fleet' %} -{%- set ip = salt['pillar.get']('node:mainip', '') %} -{%- endif %} -{%- set FLEET_NODE = salt['pillar.get']('static:fleet_node') %} -{%- set FLEET_NODE_IP = salt['pillar.get']('static:fleet_ip') %} +{% if grains['role'] in ['so-eval','so-master','so-helix','so-mastersearch', 'so-standalone'] %} + {% set ip = salt['pillar.get']('static:masterip', '') %} +{% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} + {% set ip = salt['pillar.get']('node:mainip', '') %} +{% elif grains['role'] == 'so-sensor' %} + {% set ip = salt['pillar.get']('sensor:mainip', '') %} +{% elif grains['role'] == 'so-fleet' %} + {% set ip = salt['pillar.get']('node:mainip', '') %} +{% endif %} + +{% set FLEET_NODE = salt['pillar.get']('static:fleet_node') %} +{% set FLEET_NODE_IP = salt['pillar.get']('static:fleet_ip') %} # Quick Fix for Docker being difficult iptables_fix_docker: @@ -136,7 +137,7 @@ enable_wazuh_manager_1514_udp_{{ip}}: - save: True # Rules if you are a Master -{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' %} +{% if grains['role'] in ['so-master', 'so-eval', 'so-helix', 'so-mastersearch', 'so-standalone'] %} #This should be more granular iptables_allow_master_docker: iptables.insert: diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 7b7ffe60b..906a2b33c 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -15,6 +15,7 @@ {% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} {% set MASTER = salt['grains.get']('master') %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} + {% if FEATURES %} {% set FEATURES = "-features" %} {% else %} @@ -23,35 +24,21 @@ # Logstash Section - Decide which pillar to use {% if grains['role'] == 'so-sensor' %} - -{% set lsheap = salt['pillar.get']('sensor:lsheap', '') %} -{% set lsaccessip = salt['pillar.get']('sensor:lsaccessip', '') %} - + {% set lsheap = salt['pillar.get']('sensor:lsheap', '') %} + {% set lsaccessip = salt['pillar.get']('sensor:lsaccessip', '') %} {% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} -{% set lsheap = salt['pillar.get']('node:lsheap', '') %} -{% set nodetype = salt['pillar.get']('node:node_type', 'storage') %} - -{% elif grains['role'] == 'so-master' %} - -{% set lsheap = salt['pillar.get']('master:lsheap', '') %} -{% set freq = salt['pillar.get']('master:freq', '0') %} -{% set dstats = salt['pillar.get']('master:domainstats', '0') %} -{% set nodetype = salt['grains.get']('role', '') %} - + {% set lsheap = salt['pillar.get']('node:lsheap', '') %} + {% set nodetype = salt['pillar.get']('node:node_type', 'storage') %} +{% elif grains['role'] == ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %} + {% set lsheap = salt['pillar.get']('master:lsheap', '') %} + {% set freq = salt['pillar.get']('master:freq', '0') %} + {% set dstats = salt['pillar.get']('master:domainstats', '0') %} + {% set nodetype = salt['grains.get']('role', '') %} {% elif grains['role'] == 'so-helix' %} - -{% set lsheap = salt['pillar.get']('master:lsheap', '') %} -{% set freq = salt['pillar.get']('master:freq', '0') %} -{% set dstats = salt['pillar.get']('master:domainstats', '0') %} -{% set nodetype = salt['grains.get']('role', '') %} - -{% elif grains['role'] in ['so-eval','so-mastersearch'] %} - -{% set lsheap = salt['pillar.get']('master:lsheap', '') %} -{% set freq = salt['pillar.get']('master:freq', '0') %} -{% set dstats = salt['pillar.get']('master:domainstats', '0') %} -{% set nodetype = salt['grains.get']('role', '') %} - + {% set lsheap = salt['pillar.get']('master:lsheap', '') %} + {% set freq = salt['pillar.get']('master:freq', '0') %} + {% set dstats = salt['pillar.get']('master:domainstats', '0') %} + {% set nodetype = salt['grains.get']('role', '') %} {% endif %} {% set PIPELINES = salt['pillar.get']('logstash:pipelines', {}) %} diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 162fbac0c..585bfe090 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -5,7 +5,7 @@ {% set global_ca_text = [] %} {% set global_ca_server = [] %} -{% if 'master' in grains.id.split('_')|last or 'eval' in grains.id.split('_')|last %} +{% if grains.id.split('_')|last in ['master', 'eval', 'standalone'] %} {% set trusttheca_text = salt['mine.get'](grains.id, 'x509.get_pem_entries')[grains.id]['/etc/pki/ca.crt']|replace('\n', '') %} {% set ca_server = grains.id %} {% else %} @@ -50,7 +50,7 @@ m2cryptopkgs: bits: 4096 backup: True -{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' %} +{% if grains['role'] in ['so-master', 'so-eval', 'so-helix', 'so-mastersearch', 'so-standalone'] %} # Request a cert and drop it where it needs to go to be distributed /etc/pki/filebeat.crt: @@ -142,7 +142,7 @@ fbcrtlink: backup: True {% endif %} -{% if grains['role'] == 'so-sensor' or grains['role'] == 'so-master' or grains['role'] == 'so-node' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-heavynode' or grains['role'] == 'so-fleet' %} +{% if grains['role'] == in ['so-sensor', 'so-master', 'so-node', 'so-eval', 'so-helix', 'so-mastersearch', 'so-heavynode', 'so-fleet', 'so-standalone' %} fbcertdir: file.directory: From d6d27a2861cafe8ef9950000ea93f802acc023da Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 15 May 2020 10:29:24 -0400 Subject: [PATCH 09/62] avoid duplicate entries in _cluster/settings for mastersearch https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/696 --- salt/utility/bin/crossthestreams | 6 ------ 1 file changed, 6 deletions(-) diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams index 060e943a1..c8768230e 100644 --- a/salt/utility/bin/crossthestreams +++ b/salt/utility/bin/crossthestreams @@ -38,9 +38,3 @@ echo "Applying cross cluster search config..." curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SNDATA.ip }}:9300"]}}}}}' {%- endfor %} {%- endif %} - -{%- if salt['pillar.get']('mastersearchtab', {}) %} - {%- for SN, SNDATA in salt['pillar.get']('mastersearchtab', {}).items() %} -curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SNDATA.ip }}:9300"]}}}}}' - {%- endfor %} -{%- endif %} From b9456dcc9833cb305a0eb502d244a4c3df6fb7cb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 15 May 2020 10:42:13 -0400 Subject: [PATCH 10/62] [fix] Set output dir for docker registry untar --- setup/so-functions | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index aa28d2e8a..6f59c44ad 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -686,8 +686,7 @@ docker_seed_registry() { } >> "$setup_log" 2>&1 done else - cd /nsm/docker-registry/docker - tar xvf /nsm/docker-registry/docker/registry.tar >> "$setup_log" 2>&1 + tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker >> "$setup_log" 2>&1 rm /nsm/docker-registry/docker/registry.tar >> "$setup_log" 2>&1 fi From eeb0579acb00a0404b0e694faa38558aa6078b3b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 15 May 2020 10:42:52 -0400 Subject: [PATCH 11/62] [fix] Don't change settings during whiptail menus on ISO --- setup/so-setup | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 0803ceff1..1c08796ac 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -116,13 +116,7 @@ case "$setup_type" in whiptail_management_interface_dns_search fi - # Init networking so rest of install works - set_hostname_iso - set_management_interface - collect_adminuser_inputs - add_admin_user - disable_onion_user ;; 'network') whiptail_network_notice @@ -247,6 +241,15 @@ fi whiptail_make_changes +if [[ "$setup_type" == 'iso' ]]; then + # Init networking so rest of install works + set_hostname_iso + set_management_interface + + add_admin_user + disable_onion_user +fi + set_hostname 2>> "$setup_log" set_version 2>> "$setup_log" clear_master 2>> "$setup_log" From b37b993eea98eb7eeab8fa0f03bab22fe8f88a7c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 15 May 2020 10:43:20 -0400 Subject: [PATCH 12/62] [fix] Don't say we're downloading containers on iso install --- setup/so-setup | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 1c08796ac..0fa16e110 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -319,7 +319,6 @@ export percentage=0 master_pillar 2>> "$setup_log" fi - set_progress_str 16 'Running first Salt checkin' salt_firstcheckin 2>> "$setup_log" @@ -358,7 +357,12 @@ export percentage=0 set_progress_str 25 'Configuring firewall' set_initial_firewall_policy 2>> "$setup_log" - set_progress_str 26 'Downloading containers from the internet' + if [[ "$setup_type" == 'iso' ]]; then + set_progress_str 26 'Copying containers from iso' + else + set_progress_str 26 'Downloading containers from the internet' + fi + salt-call state.apply -l info registry >> "$setup_log" 2>&1 docker_seed_registry 2>> "$setup_log" # ~ 60% when finished From fdae84bb744b46c6c741ddc78594b0698e982c4a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 15 May 2020 10:56:16 -0400 Subject: [PATCH 13/62] remove = in - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/140 --- salt/elasticsearch/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index d7c017710..eae0409f7 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -22,7 +22,7 @@ {% set FEATURES = '' %} {% endif %} -{% if grains['role'] == in ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %} +{% if grains['role'] in ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %} {% set esclustername = salt['pillar.get']('master:esclustername', '') %} {% set esheap = salt['pillar.get']('master:esheap', '') %} {% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} From f4db261baf05de8dc0f9cc709e28448dbd9f631c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 15 May 2020 10:57:25 -0400 Subject: [PATCH 14/62] change elif - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/140 --- salt/elasticsearch/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index eae0409f7..4e57f1342 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -25,7 +25,7 @@ {% if grains['role'] in ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %} {% set esclustername = salt['pillar.get']('master:esclustername', '') %} {% set esheap = salt['pillar.get']('master:esheap', '') %} -{% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} +{% elif grains['role'] in ['so-node','so-heavynode'] %} {% set esclustername = salt['pillar.get']('node:esclustername', '') %} {% set esheap = salt['pillar.get']('node:esheap', '') %} {% endif %} From f3a2c1fb4dee673f6293a5251598a84592fdf3fb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 15 May 2020 11:08:10 -0400 Subject: [PATCH 15/62] add missing ] - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/140 --- salt/ssl/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 585bfe090..f7fdbd7e5 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -142,7 +142,7 @@ fbcrtlink: backup: True {% endif %} -{% if grains['role'] == in ['so-sensor', 'so-master', 'so-node', 'so-eval', 'so-helix', 'so-mastersearch', 'so-heavynode', 'so-fleet', 'so-standalone' %} +{% if grains['role'] == in ['so-sensor', 'so-master', 'so-node', 'so-eval', 'so-helix', 'so-mastersearch', 'so-heavynode', 'so-fleet', 'so-standalone'] %} fbcertdir: file.directory: From 74290eca2a0340d43846d0f02237e4f3ccdd0859 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 15 May 2020 11:20:06 -0400 Subject: [PATCH 16/62] change = to in - https://github.com/Security-Onion-Solutions/securityonion-saltstack.git --- salt/elastalert/init.sls | 2 +- salt/logstash/init.sls | 2 +- salt/ssl/init.sls | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index 8ee844c91..3b0b3fafe 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -15,7 +15,7 @@ {% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.2') %} {% set MASTER = salt['grains.get']('master') %} -{% if grains['role'] == ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %} +{% if grains['role'] in ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %} {% set esalert = salt['pillar.get']('master:elastalert', '1') %} {% set esip = salt['pillar.get']('master:mainip', '') %} {% set esport = salt['pillar.get']('master:es_port', '') %} diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 906a2b33c..ba0e015f4 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -29,7 +29,7 @@ {% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} {% set lsheap = salt['pillar.get']('node:lsheap', '') %} {% set nodetype = salt['pillar.get']('node:node_type', 'storage') %} -{% elif grains['role'] == ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %} +{% elif grains['role'] in ['so-eval','so-mastersearch', 'so-master', 'so-standalone'] %} {% set lsheap = salt['pillar.get']('master:lsheap', '') %} {% set freq = salt['pillar.get']('master:freq', '0') %} {% set dstats = salt['pillar.get']('master:domainstats', '0') %} diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index f7fdbd7e5..e3142fa22 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -142,7 +142,7 @@ fbcrtlink: backup: True {% endif %} -{% if grains['role'] == in ['so-sensor', 'so-master', 'so-node', 'so-eval', 'so-helix', 'so-mastersearch', 'so-heavynode', 'so-fleet', 'so-standalone'] %} +{% if grains['role'] in ['so-sensor', 'so-master', 'so-node', 'so-eval', 'so-helix', 'so-mastersearch', 'so-heavynode', 'so-fleet', 'so-standalone'] %} fbcertdir: file.directory: From c9614f9baf65580fc0eeb7ba49e704030dc2dab7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 15 May 2020 11:20:33 -0400 Subject: [PATCH 17/62] [fix] Only add user to SOC on masters --- setup/so-setup | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 0fa16e110..39fcae904 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -468,8 +468,10 @@ export percentage=0 set_progress_str 86 'Updating packages' update_packages 2>> "$setup_log" - set_progress_str 87 'Adding user to SOC' - add_web_user 2>> "$setup_log" + if [[ $is_master ]]; then + set_progress_str 87 'Adding user to SOC' + add_web_user 2>> "$setup_log" + fi set_progress_str 90 'Enabling checkin at boot' checkin_at_boot 2>> "$setup_log" From 692d1d72a3fee2edded639eaa860cbbe277eed75 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 15 May 2020 11:39:59 -0400 Subject: [PATCH 18/62] [fix] Change verbiage to indicate the master vs open is only for OS packages --- setup/so-whiptail | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 4ef32bf5b..354cc758d 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1012,9 +1012,9 @@ whiptail_master_updates() { local update_string update_string=$(whiptail --title "Security Onion Setup" --radiolist \ - "How would you like to download updates for your grid?:" 20 75 4 \ - "MASTER" "Master node is proxy for OS/Docker updates." ON \ - "OPEN" "Each node connect to the Internet for updates" OFF 3>&1 1>&2 2>&3 ) + "How would you like to download OS package updates for your grid?:" 20 75 4 \ + "MASTER" "Master node is proxy for updates." ON \ + "OPEN" "Each node connects to the Internet for updates" OFF 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 2f4ce40095661b25c43de14728f7605e1646f973 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 15 May 2020 11:40:37 -0400 Subject: [PATCH 19/62] [fix] Change verbiage to indicate the master vs open is only for OS packages --- setup/so-whiptail | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 354cc758d..66c0eb693 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1035,9 +1035,9 @@ whiptail_node_updates() { [ -n "$TESTING" ] && return NODEUPDATES=$(whiptail --title "Security Onion Setup" --radiolist \ - "How would you like to download updates for this node?:" 20 75 4 \ - "MASTER" "Download OS/Docker updates from the Master." ON \ - "OPEN" "Download updates directly from the Internet" OFF 3>&1 1>&2 2>&3 ) + "How would you like to download OS package updates for your grid?:" 20 75 4 \ + "MASTER" "Master node is proxy for updates." ON \ + "OPEN" "Each node connects to the Internet for updates" OFF 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus From d04f58a27ef9bf1a42e33434a9f65285da775aec Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 15 May 2020 13:10:52 -0400 Subject: [PATCH 20/62] Update README.md --- README.md | 52 +++++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 1b7661956..28b259740 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,34 @@ -## Hybrid Hunter Beta 1.2.1 - Beta 1 +## Hybrid Hunter Beta 1.3.0 - Beta 2 ### Changes: -- Full support for Ubuntu 18.04. 16.04 is no longer supported for Hybrid Hunter. -- Introduction of the Security Onion Console. Once logged in you are directly taken to the SOC. -- New authentication using Kratos. -- During install you must specify how you would like to access the SOC ui. This is for strict cookie security. -- Ability to list and delete web users from the SOC ui. -- The soremote account is now used to add nodes to the grid vs using socore. -- Community ID support for Zeek, osquery, and Suricata. You can now tie host events to connection logs! -- Elastic 7.6.1 with ECS support. -- New set of Kibana dashboards that align with ECS. -- Eval mode no longer uses Logstash for parsing (Filebeat -> ES Ingest) -- Ingest node parsing for osquery-shipped logs (osquery, WEL, Sysmon). -- Fleet standalone mode with improved Web UI & API access control. -- Improved Fleet integration support. -- Playbook now has full Windows Sigma community ruleset builtin. -- Automatic Sigma community rule updates. -- Playbook stability enhancements. -- Zeek health check. Zeek will now auto restart if a worker crashes. -- zeekctl is now managed by salt. -- Grafana dashboard improvements and cleanup. -- Moved logstash configs to pillars. -- Salt logs moved to /opt/so/log/salt. -- Strelka integrated for file-oriented detection/analysis at scale - -### Known issues: +- New Feature: Codename: "Onion Hunt". Select Hunt from the menu and start hunting down your adversaries! +- Improved ECS support. +- Complete refactor of the setup to make it easier to follow. +- Improved setup script logging to better assist on any issues. +- Setup now checks for minimal requirements during install. +- Updated Cyberchef to version 9.20.3. +- Updated Elastalert to version 0.2.4 and switched to alpine to reduce container size. +- Updated Redis to 5.0.9 and switched to alpine to reduce container size. +- Updated Salt to 2019.2.5 +- Updated Grafana to 6.7.3. +- Zeek 3.0.6 +- Suricata 4.1.8 +- Fixes so-status to now display correct containers and status. +- local.zeek is now controlled by a pillar instead of modifying the file directly. +- Renamed so-core to so-nginx and switched to alpine to reduce container size. +- Playbook now uses MySQL instead of SQLite. +- Sigma rules have all been updated. +- Kibana dashboard improvements for ECS. +- Fixed an issue where geoip was not properly parsed. +- ATT&CK Navigator is now it's own state. +- Standlone mode is now supported. +- Mastersearch previously used the same Grafana dashboard as a Search node. It now has its own dashboard that incorporates panels from the Master node and Search node dashboards. + +### Known Issues: +- The Hunt feature is currently considered "Preview" and although very useful in its current state, not everything works. We wanted to get this out as soon as possible to get the feedback from you! Let us know what you want to see! Let us know what you think we should call it! +- You cannot pivot to PCAP from Suricata alerts in Kibana or Hunt. - Updating users via the SOC ui is known to fail. To change a user, delete the user and re-add them. - Due to the move to ECS, the current Playbook plays may not alert correctly at this time. - The osquery MacOS package does not install correctly. From 9c7481a0c7da78953b3de55dd09f986a0e8eef66 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 15 May 2020 13:21:05 -0400 Subject: [PATCH 21/62] add so-status map for standalone --- salt/common/maps/standalone.map.jinja | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 salt/common/maps/standalone.map.jinja diff --git a/salt/common/maps/standalone.map.jinja b/salt/common/maps/standalone.map.jinja new file mode 100644 index 000000000..3585d0df6 --- /dev/null +++ b/salt/common/maps/standalone.map.jinja @@ -0,0 +1,21 @@ +{% set docker = { + 'containers': [ + 'so-nginx', + 'so-telegraf', + 'so-soc', + 'so-kratos', + 'so-aptcacherng', + 'so-idstools', + 'so-redis', + 'so-logstash', + 'so-elasticsearch', + 'so-curator', + 'so-kibana', + 'so-elastalert', + 'so-filebeat', + 'so-suricata', + 'so-steno', + 'so-dockerregistry', + 'so-soctopus' + ] +} %} \ No newline at end of file From 1a157eb6a556501542147c7c4c1b03e430d7bfa9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 15 May 2020 13:23:58 -0400 Subject: [PATCH 22/62] add so-standalone to agent/ossec.conf --- salt/wazuh/files/agent/ossec.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/wazuh/files/agent/ossec.conf b/salt/wazuh/files/agent/ossec.conf index c5a61e8ad..95d17bc8d 100644 --- a/salt/wazuh/files/agent/ossec.conf +++ b/salt/wazuh/files/agent/ossec.conf @@ -1,9 +1,9 @@ -{%- if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-mastersearch' %} -{%- set ip = salt['pillar.get']('static:masterip', '') %} +{%- if grains['role'] in ['so-master', 'so-eval', 'so-mastersearch', 'so-standalone'] %} + {%- set ip = salt['pillar.get']('static:masterip', '') %} {%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} -{%- set ip = salt['pillar.get']('node:mainip', '') %} + {%- set ip = salt['pillar.get']('node:mainip', '') %} {%- elif grains['role'] == 'so-sensor' %} -{%- set ip = salt['pillar.get']('sensor:mainip', '') %} + {%- set ip = salt['pillar.get']('sensor:mainip', '') %} {%- endif %}