From 6c8d6812f9090976af1a7459a93e416c7b5c7835 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 28 Feb 2020 11:11:24 -0500 Subject: [PATCH 1/2] rename INSTALL type for starting issue 136 --- setup/so-functions | 30 +++++++++++++++--------------- setup/so-setup | 15 ++++++++------- setup/so-whiptail | 6 +++--- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index b9367890d..4c0da6a9b 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -268,7 +268,7 @@ copy_master_config() { copy_minion_tmp_files() { - if [ $INSTALLTYPE == 'MASTERONLY' ] || [ $INSTALLTYPE == 'EVALMODE' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then + if [ $INSTALLTYPE == 'MASTER' ] || [ $INSTALLTYPE == 'EVAL' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then echo "Copying pillar and salt files in $TMP to /opt/so/saltstack" cp -Rv $TMP/pillar/ /opt/so/saltstack/ >> $SETUPLOG 2>&1 if [ -d $TMP/salt ] ; then @@ -409,7 +409,7 @@ docker_install() { yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo yum -y update yum -y install docker-ce python36-docker - if [ $INSTALLTYPE == 'MASTERONLY' ] || [ $INSTALLTYPE == 'EVALMODE' ]; then + if [ $INSTALLTYPE == 'MASTER' ] || [ $INSTALLTYPE == 'EVAL' ]; then docker_registry echo "Restarting Docker" >> $SETUPLOG 2>&1 systemctl restart docker @@ -423,7 +423,7 @@ docker_install() { else - if [ $INSTALLTYPE == 'MASTERONLY' ] || [ $INSTALLTYPE == 'EVALMODE' ]; then + if [ $INSTALLTYPE == 'MASTER' ] || [ $INSTALLTYPE == 'EVAL' ]; then apt-get update >> $SETUPLOG 2>&1 apt-get -y install docker-ce python3-docker >> $SETUPLOG 2>&1 docker_registry >> $SETUPLOG 2>&1 @@ -666,7 +666,7 @@ ls_heapsize() { # Determine LS Heap Size if [ $TOTAL_MEM -ge 32000 ] || [ $INSTALLTYPE == 'MASTERSEARCH' ] || [ $INSTALLTYPE == 'HEAVYNODE' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ]; then LS_HEAP_SIZE="1000m" - elif [ $INSTALLTYPE == 'EVALMODE' ]; then + elif [ $INSTALLTYPE == 'EVAL' ]; then LS_HEAP_SIZE="700m" else # If minimal RAM, then set minimal heap @@ -685,7 +685,7 @@ master_pillar() { echo " mainint: $MAININT" >> $PILLARFILE echo " esheap: $ES_HEAP_SIZE" >> $PILLARFILE echo " esclustername: {{ grains.host }}" >> $PILLARFILE - if [ $INSTALLTYPE == 'EVALMODE' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then + if [ $INSTALLTYPE == 'EVAL' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then echo " freq: 0" >> $PILLARFILE echo " domainstats: 0" >> $PILLARFILE echo " ls_pipeline_batch_size: 125" >> $PILLARFILE @@ -877,7 +877,7 @@ saltify() { if [ $OS == 'centos' ]; then ADDUSER=adduser - if [ $INSTALLTYPE == 'MASTERONLY' ] || [ $INSTALLTYPE == 'EVALMODE' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then + if [ $INSTALLTYPE == 'MASTER' ] || [ $INSTALLTYPE == 'EVAL' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then reserve_group_ids yum -y install wget https://repo.saltstack.com/py3/redhat/salt-py3-repo-latest-2.el7.noarch.rpm cp /etc/yum.repos.d/salt-py3-latest.repo /etc/yum.repos.d/salt-py3-2019-2.repo @@ -1038,7 +1038,7 @@ EOF yum -y update exclude=salt* systemctl enable salt-minion - if [ $INSTALLTYPE == 'MASTERONLY' ] || [ $INSTALLTYPE == 'EVALMODE' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then + if [ $INSTALLTYPE == 'MASTER' ] || [ $INSTALLTYPE == 'EVAL' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then yum -y install salt-master-2019.2.3 python3 python36-m2crypto salt-minion-2019.2.3 python36-dateutil python36-mysql python36-docker systemctl enable salt-master else @@ -1058,7 +1058,7 @@ EOF UVER=$(grep VERSION_ID /etc/os-release | awk -F '[ "]' '{print $2}') # Nasty hack but required for now - if [ $INSTALLTYPE == 'MASTERONLY' ] || [ $INSTALLTYPE == 'EVALMODE' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then + if [ $INSTALLTYPE == 'MASTER' ] || [ $INSTALLTYPE == 'EVAL' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then # Install the repo for salt wget --inet4-only -O - https://repo.saltstack.com/apt/ubuntu/$UVER/amd64/latest/SALTSTACK-GPG-KEY.pub | apt-key add - @@ -1111,7 +1111,7 @@ EOF salt_checkin() { # Master State to Fix Mine Usage - if [ $INSTALLTYPE == 'MASTERONLY' ] || [ $INSTALLTYPE == 'EVALMODE' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then + if [ $INSTALLTYPE == 'MASTER' ] || [ $INSTALLTYPE == 'EVAL' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then echo "Building Certificate Authority" salt-call state.apply ca >> $SETUPLOG 2>&1 echo " *** Restarting Salt to fix any SSL errors. ***" @@ -1237,7 +1237,7 @@ set_hostname() { echo $HOSTNAME > /etc/hostname HOSTNAME=$(cat /etc/hostname) MINION_ID=$(echo $HOSTNAME | awk -F. {'print $1'}) - if [ $INSTALLTYPE != 'MASTERONLY' ] || [ $INSTALLTYPE != 'EVALMODE' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then + if [ $INSTALLTYPE != 'MASTER' ] || [ $INSTALLTYPE != 'EVAL' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then if [[ $TESTHOST = *"not found"* ]] || [[ $TESTHOST = *"connection timed out"* ]]; then if ! grep -q $MSRVIP /etc/hosts; then echo "$MSRVIP $MSRV" >> /etc/hosts @@ -1259,18 +1259,18 @@ set_hostname_iso() { set_initial_firewall_policy() { get_main_ip - if [ $INSTALLTYPE == 'MASTERONLY' ]; then + if [ $INSTALLTYPE == 'MASTER' ]; then printf " - $MAINIP\n" >> /opt/so/saltstack/pillar/firewall/minions.sls printf " - $MAINIP\n" >> /opt/so/saltstack/pillar/firewall/masterfw.sls /opt/so/saltstack/pillar/data/addtotab.sh mastertab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM fi - if [ $INSTALLTYPE == 'EVALMODE' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then + if [ $INSTALLTYPE == 'EVAL' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then printf " - $MAINIP\n" >> /opt/so/saltstack/pillar/firewall/minions.sls printf " - $MAINIP\n" >> /opt/so/saltstack/pillar/firewall/masterfw.sls printf " - $MAINIP\n" >> /opt/so/saltstack/pillar/firewall/forward_nodes.sls printf " - $MAINIP\n" >> /opt/so/saltstack/pillar/firewall/search_nodes.sls - if [ $INSTALLTYPE == 'EVALMODE' ]; then + if [ $INSTALLTYPE == 'EVAL' ]; then /opt/so/saltstack/pillar/data/addtotab.sh evaltab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM bond0 elif [ $INSTALLTYPE == 'MASTERSEARCH' ]; then /opt/so/saltstack/pillar/data/addtotab.sh mastersearchtab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM @@ -1283,7 +1283,7 @@ set_initial_firewall_policy() { printf " - $MAINIP\n" >> /opt/so/saltstack/pillar/firewall/forward_nodes.sls fi - if [ $INSTALLTYPE == 'SENSORONLY' ]; then + if [ $INSTALLTYPE == 'SENSOR' ]; then ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh sensorstab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM bond0 @@ -1336,7 +1336,7 @@ set_management_interface() { set_node_type() { # Determine the node type based on whiplash choice - if [ $INSTALLTYPE == 'SEARCHNODE' ] || [ $INSTALLTYPE == 'EVALMODE' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ] || [ $INSTALLTYPE == 'HEAVYNODE' ] ; then + if [ $INSTALLTYPE == 'SEARCHNODE' ] || [ $INSTALLTYPE == 'EVAL' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ] || [ $INSTALLTYPE == 'HEAVYNODE' ] ; then NODETYPE='search' fi if [ $INSTALLTYPE == 'PARSINGNODE' ]; then diff --git a/setup/so-setup b/setup/so-setup index 6d86a0b7a..8c4ecae00 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -31,7 +31,6 @@ fi # Global Variables HOSTNAME=$(cat /etc/hostname) -MINION_ID=$(echo $HOSTNAME | awk -F. {'print $1'}) TOTAL_MEM=`grep MemTotal /proc/meminfo | awk '{print $2}' | sed -r 's/.{3}$//'` NICS=$(ip link | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2 " \"" "Interface" "\"" " OFF"}') CPUCORES=$(cat /proc/cpuinfo | grep processor | wc -l) @@ -120,6 +119,8 @@ if (whiptail_you_sure) ; then # What kind of install are we doing? whiptail_install_type + SHORTNAME=$(echo $HOSTNAME | awk -F. {'print $1'}) + MINION_ID=$(echo $SHORTNAME'_'$INSTALLTYPE | tr '[:upper:]' '[:lower:]') # How do we want to handle OS patching? manual, auto or scheduled days and hours whiptail_patch_schedule @@ -252,7 +253,7 @@ if (whiptail_you_sure) ; then #################### ## Master ## #################### - if [ $INSTALLTYPE == 'MASTERONLY' ]; then + if [ $INSTALLTYPE == 'MASTER' ]; then # Would you like to do an advanced install? whiptail_master_adv @@ -437,7 +438,7 @@ if (whiptail_you_sure) ; then ## Sensor ## #################### - if [ $INSTALLTYPE == 'SENSORONLY' ]; then + if [ $INSTALLTYPE == 'SENSOR' ]; then filter_unused_nics whiptail_bond_nics whiptail_management_server @@ -520,12 +521,12 @@ if (whiptail_you_sure) ; then ## Eval Mode or Master Search ## ####################################### - if [ $INSTALLTYPE == 'EVALMODE' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then + if [ $INSTALLTYPE == 'EVAL' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then # Filter out the management NIC filter_unused_nics - if [ $INSTALLTYPE == 'EVALMODE' ]; then + if [ $INSTALLTYPE == 'EVAL' ]; then TYPE='eval' # Select which NICs are in the bond whiptail_bond_nics @@ -585,7 +586,7 @@ if (whiptail_you_sure) ; then add_socore_user_master { sleep 0.5 - if [ $INSTALLTYPE == 'EVALMODE' ]; then + if [ $INSTALLTYPE == 'EVAL' ]; then echo -e "XXX\n0\nCreating Bond Interface... \nXXX" create_sensor_bond >> $SETUPLOG 2>&1 fi @@ -660,7 +661,7 @@ if (whiptail_you_sure) ; then echo -e "XXX\n45\nInstalling Kibana... \nXXX" salt-call state.apply kibana >> $SETUPLOG 2>&1 - if [ $INSTALLTYPE == 'EVALMODE' ]; then + if [ $INSTALLTYPE == 'EVAL' ]; then echo -e "XXX\n50\nInstalling pcap... \nXXX" salt-call state.apply pcap >> $SETUPLOG 2>&1 echo -e "XXX\n52\nInstalling Suricata... \nXXX" diff --git a/setup/so-whiptail b/setup/so-whiptail index 59db1b29c..e828d39dc 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -249,10 +249,10 @@ whiptail_install_type() { # What kind of install are we doing? INSTALLTYPE=$(whiptail --title "Security Onion Setup" --radiolist \ "Choose Install Type:" 20 75 13 \ - "SENSORONLY" "Create a forward only sensor" ON \ + "SENSOR" "Create a forward only sensor" ON \ "SEARCHNODE" "Add a Search Node with parsing" OFF \ - "MASTERONLY" "Start a new grid" OFF \ - "EVALMODE" "Evaluate all the things" OFF \ + "MASTER" "Start a new grid" OFF \ + "EVAL" "Evaluate all the things" OFF \ "MASTERSEARCH" "Master + Search Node" OFF \ "HEAVYNODE" "Sensor + Search Node" OFF \ "HELIXSENSOR" "Connect this sensor to FireEye Helix" OFF \ From 804a87eb21a0a9038cf7680df82279420a0ec176 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 12 Mar 2020 17:42:32 -0400 Subject: [PATCH 2/2] append role to minion id and use it to target in top - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/136 --- pillar/top.sls | 16 ++++++++-------- salt/ssl/init.sls | 19 +++++++++++++------ salt/top.sls | 28 ++++++++++++++-------------- setup/so-functions | 3 +-- setup/so-setup | 1 + 5 files changed, 37 insertions(+), 30 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index 85f28b22a..9b1d60035 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -3,19 +3,19 @@ base: - patch.needs_restarting - docker.config - 'G@role:so-mastersearch or G@role:so-heavynode': + '*_mastersearch or *_heavynode': - match: compound - logstash - logstash.master - logstash.search - 'G@role:so-sensor': + '*_sensor': - static - firewall.* - brologs - minions.{{ grains.id }} - 'G@role:so-master or G@role:so-mastersearch': + '*_master or *_mastersearch': - match: compound - static - firewall.* @@ -23,11 +23,11 @@ base: - auth - minions.{{ grains.id }} - 'G@role:so-master': + '*_master': - logstash - logstash.master - 'G@role:so-eval': + '*_eval': - static - firewall.* - data.* @@ -37,18 +37,18 @@ base: - logstash.eval - minions.{{ grains.id }} - 'G@role:so-node': + '*_node': - static - firewall.* - minions.{{ grains.id }} - 'G@role:so-heavynode': + '*_heavynode': - static - firewall.* - brologs - minions.{{ grains.id }} - 'G@role:so-helix': + '*_helix': - static - firewall.* - fireeye diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index de0d5d61f..dfaf131b9 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -1,17 +1,24 @@ {% set master = salt['grains.get']('master') %} -{% set master_minion_id = master.split(".")[0] %} -{%- set masterip = salt['pillar.get']('static:masterip', '') -%} +{% set masterip = salt['pillar.get']('static:masterip', '') %} +{% set global_ca_text = [] %} +{% set global_ca_server = [] %} -{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %} +{% if 'master' in grains.id.split('_')|last or 'eval' in grains.id.split('_')|last %} {% set trusttheca_text = salt['mine.get'](grains.id, 'x509.get_pem_entries')[grains.id]['/etc/pki/ca.crt']|replace('\n', '') %} {% set ca_server = grains.id %} {% else %} - {% set trusttheca_text = salt['mine.get'](master_minion_id, 'x509.get_pem_entries')[master_minion_id]['/etc/pki/ca.crt']|replace('\n', '') %} - {% set ca_server = master_minion_id %} + {% set x509dict = salt['mine.get']('*', 'x509.get_pem_entries') %} + {% for host in x509dict %} + {% if 'master' in host.split('_')|last %} + {% do global_ca_text.append(x509dict[host].get('/etc/pki/ca.crt')|replace('\n', '')) %} + {% do global_ca_server.append(host) %} + {% endif %} + {% endfor %} + {% set trusttheca_text = global_ca_text[0] %} + {% set ca_server = global_ca_server[0] %} {% endif %} # Trust the CA - trusttheca: x509.pem_managed: - name: /etc/ssl/certs/intca.crt diff --git a/salt/top.sls b/salt/top.sls index 4af085ae1..f8f1aae40 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -11,7 +11,7 @@ base: - patch.os.schedule - motd - 'G@role:so-helix': + '*_helix': - ca - ssl - registry @@ -26,7 +26,7 @@ base: - filebeat - schedule - 'G@role:so-sensor': + '*_sensor': - ca - ssl - common @@ -43,7 +43,7 @@ base: {%- endif %} - schedule - 'G@role:so-eval': + '*_eval': - ca - ssl - registry @@ -89,7 +89,7 @@ base: {%- endif %} - 'G@role:so-master': + '*_master': - ca - ssl - registry @@ -133,8 +133,8 @@ base: # Search node logic - 'G@role:so-node and I@node:node_type:parser': - - match: pillar + '*_node and I@node:node_type:parser': + - match: compound - common - firewall - logstash @@ -143,8 +143,8 @@ base: {%- endif %} - schedule - 'G@role:so-node and I@node:node_type:hot': - - match: pillar + '*_node and I@node:node_type:hot': + - match: compound - common - firewall - logstash @@ -155,8 +155,8 @@ base: {%- endif %} - schedule - 'G@role:so-node and I@node:node_type:warm': - - match: pillar + '*_node and I@node:node_type:warm': + - match: compound - common - firewall - elasticsearch @@ -165,7 +165,7 @@ base: {%- endif %} - schedule - 'G@role:so-node and I@node:node_type:search': + '*_node and I@node:node_type:search': - match: compound - ca - ssl @@ -183,7 +183,7 @@ base: {%- endif %} - schedule - 'G@role:mastersensor': + '*_mastersensor': - common - firewall - sensor @@ -194,7 +194,7 @@ base: {%- endif %} - schedule - 'G@role:so-mastersearch': + '*_mastersearch': - ca - ssl - registry @@ -238,7 +238,7 @@ base: - domainstats {%- endif %} - 'G@role:so-heavynode': + '*_heavynode': - ca - ssl - common diff --git a/setup/so-functions b/setup/so-functions index 4c0da6a9b..579f80336 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1236,7 +1236,6 @@ set_hostname() { echo "::1 localhost localhost.localdomain localhost6 localhost6.localdomain6" >> /etc/hosts echo $HOSTNAME > /etc/hostname HOSTNAME=$(cat /etc/hostname) - MINION_ID=$(echo $HOSTNAME | awk -F. {'print $1'}) if [ $INSTALLTYPE != 'MASTER' ] || [ $INSTALLTYPE != 'EVAL' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then if [[ $TESTHOST = *"not found"* ]] || [[ $TESTHOST = *"connection timed out"* ]]; then if ! grep -q $MSRVIP /etc/hosts; then @@ -1273,7 +1272,7 @@ set_initial_firewall_policy() { if [ $INSTALLTYPE == 'EVAL' ]; then /opt/so/saltstack/pillar/data/addtotab.sh evaltab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM bond0 elif [ $INSTALLTYPE == 'MASTERSEARCH' ]; then - /opt/so/saltstack/pillar/data/addtotab.sh mastersearchtab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM + /opt/so/saltstack/pillar/data/addtotab.sh nodestab $MINION_ID $MAINIP $CPUCORES $RANDOMUID $MAININT $FSROOT $FSNSM fi fi diff --git a/setup/so-setup b/setup/so-setup index 8c4ecae00..d3956a845 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -121,6 +121,7 @@ if (whiptail_you_sure) ; then whiptail_install_type SHORTNAME=$(echo $HOSTNAME | awk -F. {'print $1'}) MINION_ID=$(echo $SHORTNAME'_'$INSTALLTYPE | tr '[:upper:]' '[:lower:]') + echo "MINION_ID = $MINION_ID" >> $SETUPLOG 2>&1 # How do we want to handle OS patching? manual, auto or scheduled days and hours whiptail_patch_schedule