From 479292ce3937f7022379c80c6fef3715b46055a9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 22 Oct 2018 13:34:07 -0400 Subject: [PATCH 01/67] Setup Script - Centos Support changes --- VERSION | 1 + so-setup-network.sh | 51 ++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 VERSION diff --git a/VERSION b/VERSION new file mode 100644 index 000000000..7dea76edb --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.0.1 diff --git a/so-setup-network.sh b/so-setup-network.sh index 8b39cf6f4..c51d44529 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -347,6 +347,12 @@ install_master() { # Install the salt master package if [ $OS == 'centos' ]; then yum -y install salt-master + + # Create a place for the keys for Ubuntu minions + mkdir -p /opt/so/gpg + wget --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/apt/ubuntu/$UVER/amd64/latest/SALTSTACK-GPG-KEY.pub + wget --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg + else apt-get install -y salt-master fi @@ -457,9 +463,48 @@ saltify() { else if [ $MASTERUPDATES == 'MASTER' ]; then - yum -y install wget - export http_proxy=http://$MSRV:3142; wget -O $TMP/salt-repo-latest-2.el7.noarch.rpm http://repo.saltstack.com/yum/redhat/salt-repo-latest-2.el7.noarch.rpm - yum -y install $TMP/salt-repo-latest-2.el7.noarch.rpm + + # Create the GPG Public Key for the Salt Repo + echo "-----BEGIN PGP PUBLIC KEY BLOCK-----" > /etc/pki/rpm-gpg/saltstack-signing-key + echo "Version: GnuPG v2.0.22 (GNU/Linux)" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "mQENBFOpvpgBCADkP656H41i8fpplEEB8IeLhugyC2rTEwwSclb8tQNYtUiGdna9" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "m38kb0OS2DDrEdtdQb2hWCnswxaAkUunb2qq18vd3dBvlnI+C4/xu5ksZZkRj+fW" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "tArNR18V+2jkwcG26m8AxIrT+m4M6/bgnSfHTBtT5adNfVcTHqiT1JtCbQcXmwVw" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "WbqS6v/LhcsBE//SHne4uBCK/GHxZHhQ5jz5h+3vWeV4gvxS3Xu6v1IlIpLDwUts" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "kT1DumfynYnnZmWTGc6SYyIFXTPJLtnoWDb9OBdWgZxXfHEcBsKGha+bXO+m2tHA" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "gNneN9i5f8oNxo5njrL8jkCckOpNpng18BKXABEBAAG0MlNhbHRTdGFjayBQYWNr" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "YWdpbmcgVGVhbSA8cGFja2FnaW5nQHNhbHRzdGFjay5jb20+iQE4BBMBAgAiBQJT" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "qb6YAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAOCKFJ3le/vhkqB/0Q" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "WzELZf4d87WApzolLG+zpsJKtt/ueXL1W1KA7JILhXB1uyvVORt8uA9FjmE083o1" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "yE66wCya7V8hjNn2lkLXboOUd1UTErlRg1GYbIt++VPscTxHxwpjDGxDB1/fiX2o" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "nK5SEpuj4IeIPJVE/uLNAwZyfX8DArLVJ5h8lknwiHlQLGlnOu9ulEAejwAKt9CU" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "4oYTszYM4xrbtjB/fR+mPnYh2fBoQO4d/NQiejIEyd9IEEMd/03AJQBuMux62tjA" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "/NwvQ9eqNgLw9NisFNHRWtP4jhAOsshv1WW+zPzu3ozoO+lLHixUIz7fqRk38q8Q" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "9oNR31KvrkSNrFbA3D89uQENBFOpvpgBCADJ79iH10AfAfpTBEQwa6vzUI3Eltqb" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "9aZ0xbZV8V/8pnuU7rqM7Z+nJgldibFk4gFG2bHCG1C5aEH/FmcOMvTKDhJSFQUx" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "uhgxttMArXm2c22OSy1hpsnVG68G32Nag/QFEJ++3hNnbyGZpHnPiYgej3FrerQJ" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "zv456wIsxRDMvJ1NZQB3twoCqwapC6FJE2hukSdWB5yCYpWlZJXBKzlYz/gwD/Fr" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "GL578WrLhKw3UvnJmlpqQaDKwmV2s7MsoZogC6wkHE92kGPG2GmoRD3ALjmCvN1E" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "PsIsQGnwpcXsRpYVCoW7e2nW4wUf7IkFZ94yOCmUq6WreWI4NggRcFC5ABEBAAGJ" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "AR8EGAECAAkFAlOpvpgCGwwACgkQDgihSd5Xv74/NggA08kEdBkiWWwJZUZEy7cK" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "WWcgjnRuOHd4rPeT+vQbOWGu6x4bxuVf9aTiYkf7ZjVF2lPn97EXOEGFWPZeZbH4" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "vdRFH9jMtP+rrLt6+3c9j0M8SIJYwBL1+CNpEC/BuHj/Ra/cmnG5ZNhYebm76h5f" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "T9iPW9fFww36FzFka4VPlvA4oB7ebBtquFg3sdQNU/MmTVV4jPFWXxh4oRDDR+8N" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "1bcPnbB11b5ary99F/mqr7RgQ+YFF0uKRE3SKa7a+6cIuHEZ7Za+zhPaQlzAOZlx" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "fuBmScum8uQTrEF5+Um5zkwC7EXTdH1co/+/V/fpOtxIg4XO4kcugZefVm5ERfVS" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "MA==" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "=dtMN" >> /etc/pki/rpm-gpg/saltstack-signing-key + echo "-----END PGP PUBLIC KEY BLOCK-----" >> /etc/pki/rpm-gpg/saltstack-signing-key + + # Proxy is hating on me.. Lets just set it manually + echo "[salt-latest]" > /etc/yum.repos.d/salt-latest.repo + echo "name=SaltStack Latest Release Channel for RHEL/Centos $releasever" >> /etc/yum.repos.d/salt-latest.repo + echo "baseurl=https://repo.saltstack.com/yum/redhat/7/$basearch/latest" >> /etc/yum.repos.d/salt-latest.repo + echo "failovermethod=priority" >> /etc/yum.repos.d/salt-latest.repo + echo "enabled=1" >> /etc/yum.repos.d/salt-latest.repo + echo "gpgcheck=1" >> /etc/yum.repos.d/salt-latest.repo + echo "gpgkey=file:///etc/pki/rpm-gpg/saltstack-signing-key" >> /etc/yum.repos.d/salt-latest.repo else yum -y install https://repo.saltstack.com/yum/redhat/salt-repo-latest-2.el7.noarch.rpm fi From c8249f363c475f9ccd14bf65551c3bdf688f9330 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 22 Oct 2018 14:07:04 -0400 Subject: [PATCH 02/67] Setup Script - Centos Support changes --- salt/master/files/acng/acng.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/master/files/acng/acng.conf b/salt/master/files/acng/acng.conf index 581beffd0..ed84d413b 100644 --- a/salt/master/files/acng/acng.conf +++ b/salt/master/files/acng/acng.conf @@ -46,6 +46,7 @@ MaxConThreads: 120 # signatures with which clients begin their downloads (WfilePattern) # VfilePatternEx: (metalink\?repo=[0-9a-zA-Z-]+&arch=[0-9a-zA-Z_-]+|/\?release=[0-9]+&arch=|repodata/.*\.(xml|sqlite)\.(gz|bz2)|APKINDEX.tar.gz|filelists\.xml\.gz|filelists\.sqlite\.bz2|repomd\.xml|packages\.[a-zA-Z][a-zA-Z]\.gz) +VfilePatternEx: ^(/\?release=[0-9]+&arch=.*|.*/RPM-GPG-KEY-examplevendor)$ PfilePatternEx: (/dists/.*/by-hash/.*|\.tgz|\.tar|\.xz|\.bz2|\.rpm|\.apk)$ # WfilePatternEx: # SPfilePatternEx: @@ -78,7 +79,7 @@ RedirMax: 6 VfileUseRangeOps: 0 # PassThroughPattern: private-ppa\.launchpad\.net:443$ # PassThroughPattern: .* # this would allow CONNECT to everything -PassThroughPattern: (download\.docker\.com:443|repo\.saltstack\.com:443|yum\.dockerproject\.org:443|download\.docker\.com:443|registry\.npmjs\.org:443|registry\.yarnpkg\.com:443)$ # yarn/npm pkg, cant to http :/ +PassThroughPattern: (download\.docker\.com:443|mirrors\.fedoraproject\.org:443|repo\.saltstack\.com:443|yum\.dockerproject\.org:443|download\.docker\.com:443|registry\.npmjs\.org:443|registry\.yarnpkg\.com:443)$ # yarn/npm pkg, cant to http :/ # ResponseFreezeDetectTime: 500 # ReuseConnections: 1 # PipelineDepth: 255 @@ -87,4 +88,4 @@ PassThroughPattern: (download\.docker\.com:443|repo\.saltstack\.com:443|yum\.doc # OptProxyTimeout: -1 # MaxDlSpeed: 500 # MaxInresponsiveDlSize: 64000 -# BadRedirDetectMime: text/html \ No newline at end of file +# BadRedirDetectMime: text/html From 195bfe87d2554d7e2482618530a729bc6f15aaf7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 22 Oct 2018 14:33:59 -0400 Subject: [PATCH 03/67] Setup Script - Fix SSH multiple logins --- so-setup-network.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index c51d44529..74ea62825 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -35,7 +35,7 @@ accept_salt_key_local() { accept_salt_key_remote() { # Accept the key remotely so the device can check in - ssh -i ~/.ssh/so.key socore@$MSRV sudo salt-key -a $HOSTNAME -y + ssh -i /root/.ssh/so.key socore@$MSRV sudo salt-key -a $HOSTNAME -y } @@ -131,18 +131,18 @@ copy_minion_pillar() { # Copy over the pillar echo "Copying the pillar over" - scp -i ~/.ssh/so.key $TMP/$HOSTNAME.sls socore@$MSRV:/opt/so/saltstack/pillar/$TYPE/$HOSTNAME.sls + scp -i /root/.ssh/so.key $TMP/$HOSTNAME.sls socore@$MSRV:/opt/so/saltstack/pillar/$TYPE/$HOSTNAME.sls } copy_ssh_key() { # Generate SSH key - mkdir -p ~/.ssh - cat /dev/zero | ssh-keygen -f ~/.ssh/so.key -t rsa -q -N "" - chown -R $SUDO_USER:$SUDO_USER ~/.ssh + mkdir -p /root/.ssh + cat /dev/zero | ssh-keygen -f /root/.ssh/so.key -t rsa -q -N "" + chown -R $SUDO_USER:$SUDO_USER /root/.ssh #Copy the key over to the master - ssh-copy-id -f -i ~/.ssh/so.key socore@$MSRV + ssh-copy-id -f -i /root/.ssh/so.key socore@$MSRV } @@ -679,14 +679,14 @@ set_initial_firewall_policy() { fi if [ $INSTALLTYPE == 'SENSORONLY' ]; then - ssh -i ~/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP - ssh -i ~/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP + ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP + ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP fi if [ $INSTALLTYPE == 'STORAGENODE' ]; then - ssh -i ~/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP - ssh -i ~/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh storage_nodes $MAINIP - ssh -i ~/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $HOSTNAME $MAINIP + ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP + ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh storage_nodes $MAINIP + ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $HOSTNAME $MAINIP fi if [ $INSTALLTYPE == 'PARSINGNODE' ]; then From f16507ff92469d2cc6c71131f431662596924da7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 22 Oct 2018 15:05:50 -0400 Subject: [PATCH 04/67] Setup Script - Fix SSH multiple logins --- so-setup-network.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 74ea62825..695e02662 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -35,7 +35,7 @@ accept_salt_key_local() { accept_salt_key_remote() { # Accept the key remotely so the device can check in - ssh -i /root/.ssh/so.key socore@$MSRV sudo salt-key -a $HOSTNAME -y + ssh -v -i /root/.ssh/so.key socore@$MSRV sudo salt-key -a $HOSTNAME -y } @@ -131,7 +131,7 @@ copy_minion_pillar() { # Copy over the pillar echo "Copying the pillar over" - scp -i /root/.ssh/so.key $TMP/$HOSTNAME.sls socore@$MSRV:/opt/so/saltstack/pillar/$TYPE/$HOSTNAME.sls + scp -v -i /root/.ssh/so.key $TMP/$HOSTNAME.sls socore@$MSRV:/opt/so/saltstack/pillar/$TYPE/$HOSTNAME.sls } @@ -142,7 +142,7 @@ copy_ssh_key() { cat /dev/zero | ssh-keygen -f /root/.ssh/so.key -t rsa -q -N "" chown -R $SUDO_USER:$SUDO_USER /root/.ssh #Copy the key over to the master - ssh-copy-id -f -i /root/.ssh/so.key socore@$MSRV + ssh-copy-id -v -f -i /root/.ssh/so.key socore@$MSRV } @@ -679,14 +679,14 @@ set_initial_firewall_policy() { fi if [ $INSTALLTYPE == 'SENSORONLY' ]; then - ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP - ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP + ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP + ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP fi if [ $INSTALLTYPE == 'STORAGENODE' ]; then - ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP - ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh storage_nodes $MAINIP - ssh -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $HOSTNAME $MAINIP + ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP + ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh storage_nodes $MAINIP + ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $HOSTNAME $MAINIP fi if [ $INSTALLTYPE == 'PARSINGNODE' ]; then From f1e03bbaaeeae741216e8b3df91adbf5c1c28e54 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 22 Oct 2018 15:34:07 -0400 Subject: [PATCH 05/67] Setup Script - Fix SSH multiple logins --- so-setup-network.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 695e02662..d1d2b0b97 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -142,7 +142,7 @@ copy_ssh_key() { cat /dev/zero | ssh-keygen -f /root/.ssh/so.key -t rsa -q -N "" chown -R $SUDO_USER:$SUDO_USER /root/.ssh #Copy the key over to the master - ssh-copy-id -v -f -i /root/.ssh/so.key socore@$MSRV + ssh-copy-id -f -i /root/.ssh/so.key socore@$MSRV } From 0b11d043070a83b82fcf7564962557baa7aba08a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 22 Oct 2018 15:57:25 -0400 Subject: [PATCH 06/67] Setup Script - Fix SSH multiple logins --- so-setup-network.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index d1d2b0b97..64f94248e 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -35,7 +35,7 @@ accept_salt_key_local() { accept_salt_key_remote() { # Accept the key remotely so the device can check in - ssh -v -i /root/.ssh/so.key socore@$MSRV sudo salt-key -a $HOSTNAME -y + ssh -v -i /root/.ssh/so.key.pub socore@$MSRV sudo salt-key -a $HOSTNAME -y } @@ -131,7 +131,7 @@ copy_minion_pillar() { # Copy over the pillar echo "Copying the pillar over" - scp -v -i /root/.ssh/so.key $TMP/$HOSTNAME.sls socore@$MSRV:/opt/so/saltstack/pillar/$TYPE/$HOSTNAME.sls + scp -v -i /root/.ssh/so.key.pub $TMP/$HOSTNAME.sls socore@$MSRV:/opt/so/saltstack/pillar/$TYPE/$HOSTNAME.sls } @@ -139,10 +139,10 @@ copy_ssh_key() { # Generate SSH key mkdir -p /root/.ssh - cat /dev/zero | ssh-keygen -f /root/.ssh/so.key -t rsa -q -N "" + cat /dev/zero | ssh-keygen -f /root/.ssh/so.key.pub -t rsa -q -N "" chown -R $SUDO_USER:$SUDO_USER /root/.ssh #Copy the key over to the master - ssh-copy-id -f -i /root/.ssh/so.key socore@$MSRV + ssh-copy-id -f -i /root/.ssh/so.key.pub socore@$MSRV } @@ -350,7 +350,7 @@ install_master() { # Create a place for the keys for Ubuntu minions mkdir -p /opt/so/gpg - wget --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/apt/ubuntu/$UVER/amd64/latest/SALTSTACK-GPG-KEY.pub + wget --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/apt/ubuntu/16.04/amd64/latest/SALTSTACK-GPG-KEY.pub wget --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg else @@ -679,14 +679,14 @@ set_initial_firewall_policy() { fi if [ $INSTALLTYPE == 'SENSORONLY' ]; then - ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP - ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP + ssh -v -i /root/.ssh/so.key.pub.pub socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP + ssh -v -i /root/.ssh/so.key.pub.pub socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP fi if [ $INSTALLTYPE == 'STORAGENODE' ]; then - ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP - ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh storage_nodes $MAINIP - ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $HOSTNAME $MAINIP + ssh -v -i /root/.ssh/so.key.pub socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP + ssh -v -i /root/.ssh/so.key.pub socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh storage_nodes $MAINIP + ssh -v -i /root/.ssh/so.key.pub socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $HOSTNAME $MAINIP fi if [ $INSTALLTYPE == 'PARSINGNODE' ]; then From 0512494861ee5aa0edb5cdfa88a0e7a33e0f7cdd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 22 Oct 2018 16:21:24 -0400 Subject: [PATCH 07/67] Setup Script - Fix SSH multiple logins --- so-setup-network.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 64f94248e..353ebf134 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -679,8 +679,8 @@ set_initial_firewall_policy() { fi if [ $INSTALLTYPE == 'SENSORONLY' ]; then - ssh -v -i /root/.ssh/so.key.pub.pub socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP - ssh -v -i /root/.ssh/so.key.pub.pub socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP + ssh -v -i /root/.ssh/so.key.pub socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP + ssh -v -i /root/.ssh/so.key.pub socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP fi if [ $INSTALLTYPE == 'STORAGENODE' ]; then From a3d5522f7b5199a91cfc59548c98065fecf969ed Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 22 Oct 2018 16:51:39 -0400 Subject: [PATCH 08/67] Setup Script - Add wget --- so-setup-network.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 353ebf134..8407aa05a 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -346,7 +346,7 @@ install_master() { # Install the salt master package if [ $OS == 'centos' ]; then - yum -y install salt-master + yum -y install salt-master wget # Create a place for the keys for Ubuntu minions mkdir -p /opt/so/gpg From 0b0099c4530f029f1960bc972d2ce8b05fc2801c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 22 Oct 2018 17:00:06 -0400 Subject: [PATCH 09/67] Setup Script - Fix SSH multiple logins --- so-setup-network.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 8407aa05a..0190e5229 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -35,7 +35,7 @@ accept_salt_key_local() { accept_salt_key_remote() { # Accept the key remotely so the device can check in - ssh -v -i /root/.ssh/so.key.pub socore@$MSRV sudo salt-key -a $HOSTNAME -y + ssh -v -i /root/.ssh/so.key socore@$MSRV sudo salt-key -a $HOSTNAME -y } @@ -131,7 +131,7 @@ copy_minion_pillar() { # Copy over the pillar echo "Copying the pillar over" - scp -v -i /root/.ssh/so.key.pub $TMP/$HOSTNAME.sls socore@$MSRV:/opt/so/saltstack/pillar/$TYPE/$HOSTNAME.sls + scp -v -i /root/.ssh/so.key $TMP/$HOSTNAME.sls socore@$MSRV:/opt/so/saltstack/pillar/$TYPE/$HOSTNAME.sls } @@ -139,10 +139,10 @@ copy_ssh_key() { # Generate SSH key mkdir -p /root/.ssh - cat /dev/zero | ssh-keygen -f /root/.ssh/so.key.pub -t rsa -q -N "" + cat /dev/zero | ssh-keygen -f /root/.ssh/so.key -t rsa -q -N "" chown -R $SUDO_USER:$SUDO_USER /root/.ssh #Copy the key over to the master - ssh-copy-id -f -i /root/.ssh/so.key.pub socore@$MSRV + ssh-copy-id -f -i /root/.ssh/so.key socore@$MSRV } @@ -679,14 +679,14 @@ set_initial_firewall_policy() { fi if [ $INSTALLTYPE == 'SENSORONLY' ]; then - ssh -v -i /root/.ssh/so.key.pub socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP - ssh -v -i /root/.ssh/so.key.pub socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP + ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP + ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP fi if [ $INSTALLTYPE == 'STORAGENODE' ]; then - ssh -v -i /root/.ssh/so.key.pub socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP - ssh -v -i /root/.ssh/so.key.pub socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh storage_nodes $MAINIP - ssh -v -i /root/.ssh/so.key.pub socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $HOSTNAME $MAINIP + ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP + ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh storage_nodes $MAINIP + ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/data/addtotab.sh nodestab $HOSTNAME $MAINIP fi if [ $INSTALLTYPE == 'PARSINGNODE' ]; then From 8c9a8c871767da1d19c132cac4b5a5bd0f114570 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 24 Oct 2018 09:41:29 -0400 Subject: [PATCH 10/67] ACNG - Trying to fix the salt repo for centos --- salt/master/files/acng/acng.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/master/files/acng/acng.conf b/salt/master/files/acng/acng.conf index ed84d413b..fd4c84991 100644 --- a/salt/master/files/acng/acng.conf +++ b/salt/master/files/acng/acng.conf @@ -19,11 +19,12 @@ Remap-yarn: registry.yarnpkg.com Remap-npm: registry.npmjs.org Remap-node: nodejs.org Remap-apache: file:apache_mirrors ; file:backends_apache.us +Remap-salt: repo.saltstack.com; https://repo.saltstack.com # Remap-secdeb: security.debian.org ReportPage: acng-report.html # SocketPath:/var/run/apt-cacher-ng/socket UnbufferLogs: 1 -VerboseLog: 2 +VerboseLog: 1 ForeGround: 1 # PidFile: /var/run/apt-cacher-ng/pid # Offlinemode: 0 @@ -46,7 +47,6 @@ MaxConThreads: 120 # signatures with which clients begin their downloads (WfilePattern) # VfilePatternEx: (metalink\?repo=[0-9a-zA-Z-]+&arch=[0-9a-zA-Z_-]+|/\?release=[0-9]+&arch=|repodata/.*\.(xml|sqlite)\.(gz|bz2)|APKINDEX.tar.gz|filelists\.xml\.gz|filelists\.sqlite\.bz2|repomd\.xml|packages\.[a-zA-Z][a-zA-Z]\.gz) -VfilePatternEx: ^(/\?release=[0-9]+&arch=.*|.*/RPM-GPG-KEY-examplevendor)$ PfilePatternEx: (/dists/.*/by-hash/.*|\.tgz|\.tar|\.xz|\.bz2|\.rpm|\.apk)$ # WfilePatternEx: # SPfilePatternEx: From 724945765e80b8b7498c6ff5748db67eab692162 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 25 Oct 2018 16:10:20 -0400 Subject: [PATCH 11/67] Master Module - Change acng docker --- salt/master/init.sls | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/master/init.sls b/salt/master/init.sls index 63e6301a3..40de7b379 100644 --- a/salt/master/init.sls +++ b/salt/master/init.sls @@ -46,16 +46,15 @@ acngcopyconf: - name: /opt/so/conf/aptcacher-ng/etc/acng.conf - source: salt://master/files/acng/acng.conf -# Install the apt-cacher-ng container - TODO Create a so-docker for it +# Install the apt-cacher-ng container so-aptcacherng: docker_container.running: - - image: deployable/acng:latest-us + - image: toosmooth/so-acng:techpreview - hostname: so-aptcacherng - port_bindings: - 0.0.0.0:3142:3142 - binds: - /opt/so/conf/aptcacher-ng/cache:/var/cache/apt-cacher-ng:rw - - /opt/so/conf/aptcacher-ng/etc/acng.conf:/etc/apr-cacher-ng/acng.conf:ro - /opt/so/log/aptcacher-ng:/var/log/apt-cacher-ng:rw From 69e46946950bb3a736db7a5379067814c936e454 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 25 Oct 2018 16:50:50 -0400 Subject: [PATCH 12/67] Master Module - Change acng docker --- salt/master/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/master/init.sls b/salt/master/init.sls index 40de7b379..47c13f5ca 100644 --- a/salt/master/init.sls +++ b/salt/master/init.sls @@ -50,7 +50,7 @@ acngcopyconf: so-aptcacherng: docker_container.running: - image: toosmooth/so-acng:techpreview - - hostname: so-aptcacherng + - hostname: so-acng - port_bindings: - 0.0.0.0:3142:3142 - binds: From 64f992f1ffaf3f4bb4a5ce9a0872f534f1780e61 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 25 Oct 2018 21:24:14 -0400 Subject: [PATCH 13/67] Master Module - ACNG --- salt/master/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/master/init.sls b/salt/master/init.sls index 47c13f5ca..1e37d54c9 100644 --- a/salt/master/init.sls +++ b/salt/master/init.sls @@ -56,6 +56,7 @@ so-aptcacherng: - binds: - /opt/so/conf/aptcacher-ng/cache:/var/cache/apt-cacher-ng:rw - /opt/so/log/aptcacher-ng:/var/log/apt-cacher-ng:rw + - /opt/so/conf/aptcacher-ng/etc/acng.conf:/etc/apt-cacher-ng/acng.conf:ro # Create the config directory for the docker registry From 84703eaa4dbb92899788ca7584ff11bf23710b2a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 25 Oct 2018 21:52:39 -0400 Subject: [PATCH 14/67] Master Module - ACNG --- so-setup-network.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 0190e5229..30a6b5cce 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -499,8 +499,8 @@ saltify() { # Proxy is hating on me.. Lets just set it manually echo "[salt-latest]" > /etc/yum.repos.d/salt-latest.repo - echo "name=SaltStack Latest Release Channel for RHEL/Centos $releasever" >> /etc/yum.repos.d/salt-latest.repo - echo "baseurl=https://repo.saltstack.com/yum/redhat/7/$basearch/latest" >> /etc/yum.repos.d/salt-latest.repo + echo "name=SaltStack Latest Release Channel for RHEL/Centos \$releasever" >> /etc/yum.repos.d/salt-latest.repo + echo "baseurl=https://repo.saltstack.com/yum/redhat/7/\$basearch/latest" >> /etc/yum.repos.d/salt-latest.repo echo "failovermethod=priority" >> /etc/yum.repos.d/salt-latest.repo echo "enabled=1" >> /etc/yum.repos.d/salt-latest.repo echo "gpgcheck=1" >> /etc/yum.repos.d/salt-latest.repo From 90d55104c6cab39377252b19c567c9a0fe86a0f8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 25 Oct 2018 22:43:19 -0400 Subject: [PATCH 15/67] Filebeat Module - Fix bro logs to make them work --- salt/filebeat/etc/filebeat.yml | 2 +- salt/filebeat/init.sls | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 8f053a42b..15317921e 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -14,7 +14,7 @@ filebeat.prospectors: {%- for LOGNAME in salt['pillar.get']('brologs:enabled', '') %} - type: log paths: - - /nsm/bro/spool/{{ LOGNAME }}.log + - /nsm/bro/logs/current/{{ LOGNAME }}.log fields: type: bro_{{ LOGNAME }} fields_under_root: true diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index fcac162cd..6fb65bd63 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -59,7 +59,7 @@ so-filebeat: - binds: - /opt/so/log/filebeat:/var/log/filebeat:rw - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - - /nsm/bro/spool/manager:/nsm/bro/spool:ro + - /nsm/bro:/nsm/bro:ro - /opt/so/log/suricata:/suricata:ro - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro From 91c44eb2ff72cc89171047fd001a26ffa59c7a21 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 25 Oct 2018 22:45:08 -0400 Subject: [PATCH 16/67] Bro Module - Switch to JSON --- salt/bro/files/local.bro | 3 +++ salt/bro/files/local.bro.community | 3 +++ 2 files changed, 6 insertions(+) diff --git a/salt/bro/files/local.bro b/salt/bro/files/local.bro index 74404e704..d97fbd6f3 100644 --- a/salt/bro/files/local.bro +++ b/salt/bro/files/local.bro @@ -139,3 +139,6 @@ # Load a custom Bro policy # /opt/so/saltstack/bro/policy/custom/ on the master #@load custom/somebropolicy.bro + +# Write logs in JSON +redef LogAscii::use_json = T; diff --git a/salt/bro/files/local.bro.community b/salt/bro/files/local.bro.community index ee79789f5..40476d8e7 100644 --- a/salt/bro/files/local.bro.community +++ b/salt/bro/files/local.bro.community @@ -127,3 +127,6 @@ # Load a custom Bro policy # /opt/so/saltstack/bro/policy/custom/ on the master #@load custom/somebropolicy.bro + +# Use JSON +redef LogAscii::use_json = T; From bdb85cd28dfa3cb88cbeb050f80ace9d9b4ee117 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 26 Oct 2018 13:32:51 -0400 Subject: [PATCH 17/67] Setup Script - Fix formatting --- so-setup-network.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 30a6b5cce..4309e75f0 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -660,7 +660,7 @@ sensor_pillar() { echo " pcapbpf:" >> $TMP/$HOSTNAME.sls echo " nidsbpf:" >> $TMP/$HOSTNAME.sls echo " master: $MSRV" >> $TMP/$HOSTNAME.sls - if [ $HNSENSOR != 'inherit']; then + if [ $HNSENSOR != 'inherit' ]; then echo " hnsensor: $HNSENSOR" >> $TMP/$HOSTNAME.sls fi echo " access_key: $ACCESS_KEY" >> $TMP/$HOSTNAME.sls From fb1dc9532e60a25ad78f1baaebc9f53fb0d06b73 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 26 Oct 2018 14:31:37 -0400 Subject: [PATCH 18/67] Setup Script - Make Centos bond interface come up on boot --- so-setup-network.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 4309e75f0..7176ba964 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -161,12 +161,13 @@ create_bond() { echo "BONDING_MASTER=yes" >> /etc/sysconfig/network-scripts/ifcfg-bond0 echo "BOOTPROTO=none" >> /etc/sysconfig/network-scripts/ifcfg-bond0 echo "BONDING_OPTS=\"mode=0\"" >> /etc/sysconfig/network-scripts/ifcfg-bond0 + echo "ONBOOT=yes" # Create Bond configs for the selected monitor interface for BNIC in ${BNICS[@]}; do BONDNIC="${BNIC%\"}" BONDNIC="${BONDNIC#\"}" - sed -i 's/ONBOOT=\"no\"/ONBOOT=\"yes\"/g' /etc/sysconfig/network-scripts/ifcfg-$BONDNIC + sed -i 's/ONBOOT=no/ONBOOT=yes/g' /etc/sysconfig/network-scripts/ifcfg-$BONDNIC echo "MASTER=bond0" >> /etc/sysconfig/network-scripts/ifcfg-$BONDNIC echo "SLAVE=yes" >> /etc/sysconfig/network-scripts/ifcfg-$BONDNIC done @@ -912,7 +913,7 @@ whiptail_make_changes() { whiptail_management_server() { MSRV=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter your Master Server HOSTNAME" 10 60 XXXX 3>&1 1>&2 2>&3) + "Enter your Master Server HOSTNAME. It is CASE SENSATIVE!" 10 60 XXXX 3>&1 1>&2 2>&3) # See if it resolves. Otherwise prompt to add to host file TESTHOST=$(host $MSRV) From 32f66cb23e6dd2e3aad3e756c6de5b95fdbc6854 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 26 Oct 2018 14:35:34 -0400 Subject: [PATCH 19/67] Setup Script - Fix Spelling error --- so-setup-network.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 7176ba964..10a0292c6 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -913,7 +913,7 @@ whiptail_make_changes() { whiptail_management_server() { MSRV=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter your Master Server HOSTNAME. It is CASE SENSATIVE!" 10 60 XXXX 3>&1 1>&2 2>&3) + "Enter your Master Server HOSTNAME. It is CASE SENSITIVE!" 10 60 XXXX 3>&1 1>&2 2>&3) # See if it resolves. Otherwise prompt to add to host file TESTHOST=$(host $MSRV) From ce372b939a6cce9d096bcf7ec3558720bcbd8406 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 30 Oct 2018 10:42:07 -0400 Subject: [PATCH 20/67] Setup Script - Step 1 for Eval Mode --- so-setup-network.sh | 89 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 71 insertions(+), 18 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 10a0292c6..1f6837070 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -30,6 +30,7 @@ accept_salt_key_local() { # Accept the key locally on the master salt-key -ya $HOSTNAME + } accept_salt_key_remote() { @@ -335,6 +336,7 @@ install_cleanup() { rm -rf ./installtmp } + install_prep() { # Create a tmp space that isn't in /tmp @@ -380,7 +382,6 @@ master_pillar() { touch /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo "master:" > /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " mainip: $MAINIP" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls - echo " esaccessip: 127.0.0.1" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " esheap: $ES_HEAP_SIZE" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " esclustername: {{ grains.host }}" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls if [ $INSTALLTYPE == 'EVALMODE' ]; then @@ -400,8 +401,8 @@ master_pillar() { echo " ls_pipeline_workers: $CPUCORES" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " nids_rules: $RULESETUP" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " oinkcode: $OINKCODE" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls - echo " access_key: $ACCESS_KEY" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls - echo " access_secret: $ACCESS_SECRET" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + #echo " access_key: $ACCESS_KEY" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + #echo " access_secret: $ACCESS_SECRET" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls } @@ -432,17 +433,16 @@ minio_generate_keys() { ACCESS_SECRET=$(cat /dev/urandom | tr -cd "$charSet" | tr -d \' | tr -d \" | head -c 40) } + node_pillar() { # Create the node pillar touch $TMP/$HOSTNAME.sls echo "node:" > $TMP/$HOSTNAME.sls echo " mainip: $MAINIP" >> $TMP/$HOSTNAME.sls - echo " esaccessip: 127.0.0.1" >> $TMP/$HOSTNAME.sls echo " esheap: $NODE_ES_HEAP_SIZE" >> $TMP/$HOSTNAME.sls echo " esclustername: {{ grains.host }}" >> $TMP/$HOSTNAME.sls echo " lsheap: $NODE_LS_HEAP_SIZE" >> $TMP/$HOSTNAME.sls - echo " lsaccessip: 127.0.0.1" >> $TMP/$HOSTNAME.sls echo " ls_pipeline_workers: $LSPIPELINEWORKERS" >> $TMP/$HOSTNAME.sls echo " ls_pipeline_batch_size: $LSPIPELINEBATCH" >> $TMP/$HOSTNAME.sls echo " ls_input_threads: $LSINPUTTHREADS" >> $TMP/$HOSTNAME.sls @@ -675,15 +675,18 @@ set_initial_firewall_policy() { if [ $INSTALLTYPE == 'MASTERONLY' ]; then printf " - $MAINIP\n" >> /opt/so/saltstack/pillar/firewall/minions.sls printf " - $MAINIP\n" >> /opt/so/saltstack/pillar/firewall/masterfw.sls - - fi - if [ $INSTALLTYPE == 'SENSORONLY' ]; then + if [ $INSTALLTYPE == 'EVALMODE' ]; then + printf " - $MAINIP\n" >> /opt/so/saltstack/pillar/firewall/minions.sls + printf " - $MAINIP\n" >> /opt/so/saltstack/pillar/firewall/masterfw.sls + fi + + if [ $INSTALLTYPE == 'SENSORONLY' ]; then ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh forward_nodes $MAINIP - fi + if [ $INSTALLTYPE == 'STORAGENODE' ]; then ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh minions $MAINIP ssh -v -i /root/.ssh/so.key socore@$MSRV sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh storage_nodes $MAINIP @@ -785,7 +788,7 @@ whiptail_bro_pins() { whiptail_bro_version() { - BROVERSION=$(whiptail --title "Security Onion Setup" --radiolist "Which version of Bro would you like to use?" 20 78 4 "COMMUNITY" "Install Community Bro" ON "BRO" "Install Standard Bro" OFF 3>&1 1>&2 2>&3) + BROVERSION=$(whiptail --title "Security Onion Setup" --radiolist "Which version of Bro would you like to use?" 20 78 4 "COMMUNITY" "Install Community Bro" ON "ZEEK" "Install Zeek" OFF 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -850,8 +853,8 @@ whiptail_homenet_sensor() { "Enter your HOME_NET separated by ," 10 60 10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 3>&1 1>&2 2>&3) fi - } + whiptail_install_type() { # What kind of install are we doing? @@ -862,9 +865,8 @@ whiptail_install_type() { "STORAGENODE" "Add a Storage Hot Node with parsing" OFF \ "PARSINGNODE" "TODO Add a dedicated Parsing Node" OFF \ "HOTNODE" "TODO Add a Hot Node (Storage Node without Parsing)" OFF \ - "EVALMODE" "TODO Evaluate all the things" OFF \ - "WARMNODE" "TODO Add a Warm Node to an existing Hot or Storage node" OFF 3>&1 1>&2 2>&3 ) - + "WARMNODE" "TODO Add a Warm Node to an existing Hot or Storage node" OFF \ + "EVALMODE" "Evaluate all the things" OFF 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1065,6 +1067,7 @@ whiptail_you_sure() { whiptail --title "Security Onion Setup" --yesno "Are you sure you want to install Security Onion over the internet?" 8 78 } + ######################## ## ## ## End Functions ## @@ -1087,30 +1090,38 @@ if [ $OS == ubuntu ]; then fi # Question Time - if (whiptail_you_sure); then - # Create a dir to get started + # Create a temp dir to get started install_prep # Let folks know they need their management interface already set up. whiptail_network_notice - # Go ahead and gen the keys so we can use them for any sensor type + # Go ahead and gen the keys so we can use them for any sensor type - Disabled for now #minio_generate_keys + # What kind of install are we doing? whiptail_install_type + #################### + ## Master ## + #################### + if [ $INSTALLTYPE == 'MASTERONLY' ]; then # Pick the Management NIC whiptail_management_nic - # Choose your bro + + # Choose Zeek or Community Bro whiptail_bro_version + # Select Snort or Suricata whiptail_nids + # Snag the HOME_NET whiptail_homenet_master + # Pick your Ruleset whiptail_rule_setup @@ -1125,6 +1136,8 @@ if (whiptail_you_sure); then # Last Chance to back out whiptail_make_changes + + # Figure out the main IP address get_main_ip # Add the user so we can sit back and relax @@ -1137,39 +1150,64 @@ if (whiptail_you_sure); then echo " ** Installing Salt and Dependencies **" saltify >>~/sosetup.log 2>&1 docker_install + # Configure the Minion echo " ** Configuring Minion **" configure_minion master >>~/sosetup.log 2>&1 + + # Install the salt master echo " ** Installing Salt Master **" install_master >>~/sosetup.log 2>&1 + # Copy the data over salt_master_directories >>~/sosetup.log 2>&1 + # Update sudoers file to allow keys and firewalls to be changed update_sudoers + + # Change perms on the master dir chown_salt_master + + # Determine the ES Heap Size es_heapsize + + # Determine the Logstash Heap Size ls_heapsize + # Set the static values master_static + echo "** Generating the master pillar **" master_pillar + # Do a checkin to push the key up echo "** Pushing the key up to Master **" salt_firstcheckin >>~/sosetup.log 2>&1 + # Accept the Master Key echo "** Accepting the key on the master **" accept_salt_key_local + + # Open the firewall echo "** Setting the initial firewall policy **" set_initial_firewall_policy + # Do the big checkin but first let them know it will take a bit. salt_checkin_message salt_checkin + + # Enable salt to run a checking when the service starts checkin_at_boot + # We are done! whiptail_setup_complete fi + #################### + ## Sensor ## + #################### + if [ $INSTALLTYPE == 'SENSORONLY' ]; then whiptail_management_nic filter_nics @@ -1210,13 +1248,24 @@ if (whiptail_you_sure); then fi + ####################### + ## Eval Mode ## + ####################### + if [ $INSTALLTYPE == 'EVALMODE' ]; then whiptail_management_nic filter_nics whiptail_bond_nics whiptail_management_server whiptail_nids + whiptail_bro_version whiptail_sensor_config + NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE + NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE + LSPIPELINEWORKERS=1 + LSPIPELINEBATCH=125 + LSINPUTTHREADS=1 + LSINPUTBATCHCOUNT=125 whiptail_make_changes configure_minion copy_ssh_key @@ -1232,6 +1281,10 @@ if (whiptail_you_sure); then checkin_at_boot fi + ################### + ## Nodes ## + ################### + if [ $INSTALLTYPE == 'STORAGENODE' ] || [ $INSTALLTYPE == 'PARSINGNODE' ] || [ $INSTALLTYPE == 'HOTNODE' ] || [ $INSTALLTYPE == 'WARMNODE' ]; then whiptail_management_nic whiptail_management_server From 26217806288332682217454915869d152bf943ad Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 30 Oct 2018 13:25:21 -0400 Subject: [PATCH 21/67] Setup Script - Change top file --- salt/top.sls | 17 ++++++++++++++--- so-setup-network.sh | 13 +++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/salt/top.sls b/salt/top.sls index a6a4dd344..ba7ab520f 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -8,11 +8,22 @@ base: - bro - filebeat - 'G@role:eval': + 'G@role:so-eval': + - ca + - ssl - common - - sensor + - firewall - master - - eval + - idstools + - redis + - elasticsearch + - logstash + - kibana + - utility + - pcap + - suricata + - bro + 'G@role:so-master': - ca diff --git a/so-setup-network.sh b/so-setup-network.sh index 1f6837070..3ca75bfbf 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1253,13 +1253,26 @@ if (whiptail_you_sure); then ####################### if [ $INSTALLTYPE == 'EVALMODE' ]; then + # Select the management NIC whiptail_management_nic + + # Filter out the management NIC filter_nics + + # Select which NICs are in the bond whiptail_bond_nics + + # Set Management Server - Fix This whiptail_management_server + + # Set the NIDS to suricata whiptail_nids + whiptail_bro_version whiptail_sensor_config + # Set a bunch of stuff since this is eval + es_heapsize + ls_heapsize NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE LSPIPELINEWORKERS=1 From b30be6b6601421251a3263371c8a740ac40e5c3a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 30 Oct 2018 13:29:18 -0400 Subject: [PATCH 22/67] Setup Script - Make cancel exit on all screens --- so-setup-network.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index 3ca75bfbf..b24ad6dd6 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -838,6 +838,9 @@ whiptail_homenet_master() { HNMASTER=$(whiptail --title "Security Onion Setup" --inputbox \ "Enter your HOME_NET separated by ," 10 60 10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + } whiptail_homenet_sensor() { @@ -867,6 +870,7 @@ whiptail_install_type() { "HOTNODE" "TODO Add a Hot Node (Storage Node without Parsing)" OFF \ "WARMNODE" "TODO Add a Warm Node to an existing Hot or Storage node" OFF \ "EVALMODE" "Evaluate all the things" OFF 3>&1 1>&2 2>&3 ) + local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -946,6 +950,9 @@ whiptail_node_advanced() { "NODEBASIC" "Install Storage Node with recommended settings" ON \ "NODEADVANCED" "Advanced Node Setup" OFF 3>&1 1>&2 2>&3 ) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + } whiptail_node_es_heap() { @@ -954,6 +961,9 @@ whiptail_node_es_heap() { NODE_ES_HEAP_SIZE=$(whiptail --title "Security Onion Setup" --inputbox \ "\nEnter ES Heap Size: \n \n(Recommended value is pre-populated)" 10 60 $ES_HEAP_SIZE 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + } whiptail_node_ls_heap() { @@ -962,6 +972,9 @@ whiptail_node_ls_heap() { NODE_LS_HEAP_SIZE=$(whiptail --title "Security Onion Setup" --inputbox \ "\nEnter LogStash Heap Size: \n \n(Recommended value is pre-populated)" 10 60 $LS_HEAP_SIZE 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + } whiptail_node_ls_pipeline_worker() { @@ -969,6 +982,9 @@ whiptail_node_ls_pipeline_worker() { LSPIPELINEWORKERS=$(whiptail --title "Security Onion Setup" --inputbox \ "\nEnter LogStash Pipeline Workers: \n \n(Recommended value is pre-populated)" 10 60 $CPUCORES 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + } whiptail_node_ls_pipline_batchsize() { @@ -976,6 +992,9 @@ whiptail_node_ls_pipline_batchsize() { LSPIPELINEBATCH=$(whiptail --title "Security Onion Setup" --inputbox \ "\nEnter LogStash Pipeline Batch Size: \n \n(Default value is pre-populated)" 10 60 125 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + } whiptail_node_ls_input_threads() { @@ -983,6 +1002,9 @@ whiptail_node_ls_input_threads() { LSINPUTTHREADS=$(whiptail --title "Security Onion Setup" --inputbox \ "\nEnter LogStash Input Threads: \n \n(Default value is pre-populated)" 10 60 1 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + } whiptail_node_ls_input_batch_count() { @@ -990,6 +1012,9 @@ whiptail_node_ls_input_batch_count() { LSINPUTBATCHCOUNT=$(whiptail --title "Security Onion Setup" --inputbox \ "\nEnter LogStash Input Batch Count: \n \n(Default value is pre-populated)" 10 60 125 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + } whiptail_rule_setup() { @@ -1032,6 +1057,9 @@ whiptail_shard_count() { SHARDCOUNT=$(whiptail --title "Security Onion Setup" --inputbox \ "\nEnter ES Shard Count: \n \n(Default value is pre-populated)" 10 60 125 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + } whiptail_suricata_pins() { @@ -1051,6 +1079,9 @@ whiptail_master_updates() { "MASTER" "Have the master node act as a proxy for OS/Docker updates." ON \ "OPEN" "Have each node connect to the Internet for updates" OFF 3>&1 1>&2 2>&3 ) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + } whiptail_node_updates() { @@ -1060,6 +1091,9 @@ whiptail_node_updates() { "MASTER" "Download OS/Docker updates from the Master." ON \ "OPEN" "Download updates directly from the Internet" OFF 3>&1 1>&2 2>&3 ) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + } whiptail_you_sure() { From 18eff68dc7cee0962cc7bcafd1ab822d0aba29d3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 30 Oct 2018 16:44:13 -0400 Subject: [PATCH 23/67] Setup Script - Remove Master prompt for evalmode --- so-setup-network.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index b24ad6dd6..1d3a4da50 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -103,7 +103,7 @@ configure_minion() { echo "Configuring minion type as $TYPE" touch /etc/salt/grains echo "role: so-$TYPE" > /etc/salt/grains - if [ $TYPE == 'master' ]; then + if [ $TYPE == 'master' ] || [ $TYPE == 'eval' ]; then echo "master: $HOSTNAME" > /etc/salt/minion echo "id: $HOSTNAME" >> /etc/salt/minion else @@ -1296,9 +1296,6 @@ if (whiptail_you_sure); then # Select which NICs are in the bond whiptail_bond_nics - # Set Management Server - Fix This - whiptail_management_server - # Set the NIDS to suricata whiptail_nids @@ -1314,13 +1311,19 @@ if (whiptail_you_sure); then LSINPUTTHREADS=1 LSINPUTBATCHCOUNT=125 whiptail_make_changes - configure_minion - copy_ssh_key + # Add the user so we can sit back and relax + echo "" + echo "**** Please set a password for socore. You will use this password when setting up other Nodes/Sensors" + echo "" + add_socore_user_master create_bond saltify docker_install - configure_minion sensor + install_master + configure_minion eval copy_minion_pillar sensors + set_node_type + node_pillar salt_firstcheckin accept_salt_key_local salt_checkin_message From 879d144d54dc9e57073b67fe2c798e3438d523e5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 30 Oct 2018 16:53:04 -0400 Subject: [PATCH 24/67] Setup Script - Adding more master functions --- so-setup-network.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index 1d3a4da50..6279c9185 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1320,6 +1320,16 @@ if (whiptail_you_sure); then saltify docker_install install_master + # Copy the data over + salt_master_directories + update_sudoers + # Change perms on the master dir + chown_salt_master + # Set the static values + master_static + echo "** Generating the master pillar **" + master_pillar + configure_minion eval copy_minion_pillar sensors set_node_type From c8eff7d1bee7297efb16e2054915ac1d2abe96dd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 08:51:24 -0400 Subject: [PATCH 25/67] Setup Script - Fixes --- README.md | 4 ++-- so-setup-network.sh | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 2f89d45d6..33d594fad 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Security Onion NP Tech Preview +# Security Onion Hybrid Hunter Tech Preview Installation: @@ -8,4 +8,4 @@ cd securityonion-saltstack sudo bash so-setup-network.sh ``` -See the [FAQ](https://github.com/Security-Onion-Solutions/securityonion-saltstack/wiki/FAQ) on the TP wiki. +See the [FAQ](https://github.com/Security-Onion-Solutions/securityonion-saltstack/wiki/FAQ) on the Hybrid Hunter wiki. diff --git a/so-setup-network.sh b/so-setup-network.sh index 6279c9185..ee6b9f022 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -571,7 +571,7 @@ saltify() { salt_checkin() { # Master State to Fix Mine Usage - if [ $INSTALLTYPE == 'MASTERONLY' ]; then + if [ $INSTALLTYPE == 'MASTERONLY' ] || [ $INSTALLTYPE == 'EVALMODE' ]; then salt-call state.apply ca >>~/sosetup.log 2>&1 # salt-call state.apply ssl >>~/sosetup.log 2>&1 # salt-call state.apply common >>~/sosetup.log 2>&1 @@ -680,6 +680,8 @@ set_initial_firewall_policy() { if [ $INSTALLTYPE == 'EVALMODE' ]; then printf " - $MAINIP\n" >> /opt/so/saltstack/pillar/firewall/minions.sls printf " - $MAINIP\n" >> /opt/so/saltstack/pillar/firewall/masterfw.sls + printf " - $MAINIP\n" >> /opt/so/saltstack/pillar/firewall/forward_nodes.sls + printf " - $MAINIP\n" >> /opt/so/saltstack/pillar/firewall/storage_nodes.sls fi if [ $INSTALLTYPE == 'SENSORONLY' ]; then @@ -710,7 +712,7 @@ set_initial_firewall_policy() { set_node_type() { # Determine the node type based on whiplash choice - if [ $INSTALLTYPE == 'STORAGENODE' ]; then + if [ $INSTALLTYPE == 'STORAGENODE' ] || [ $INSTALLTYPE == 'EVALMODE' ]; then NODETYPE='storage' fi if [ $INSTALLTYPE == 'PARSINGNODE' ]; then @@ -864,8 +866,8 @@ whiptail_install_type() { INSTALLTYPE=$(whiptail --title "Security Onion Setup" --radiolist \ "Choose Install Type:" 20 78 8 \ "SENSORONLY" "Create a forward only sensor" ON \ - "MASTERONLY" "Start a new grid" OFF \ "STORAGENODE" "Add a Storage Hot Node with parsing" OFF \ + "MASTERONLY" "Start a new grid" OFF \ "PARSINGNODE" "TODO Add a dedicated Parsing Node" OFF \ "HOTNODE" "TODO Add a Hot Node (Storage Node without Parsing)" OFF \ "WARMNODE" "TODO Add a Warm Node to an existing Hot or Storage node" OFF \ @@ -1311,6 +1313,7 @@ if (whiptail_you_sure); then LSINPUTTHREADS=1 LSINPUTBATCHCOUNT=125 whiptail_make_changes + get_main_ip # Add the user so we can sit back and relax echo "" echo "**** Please set a password for socore. You will use this password when setting up other Nodes/Sensors" @@ -1329,9 +1332,7 @@ if (whiptail_you_sure); then master_static echo "** Generating the master pillar **" master_pillar - configure_minion eval - copy_minion_pillar sensors set_node_type node_pillar salt_firstcheckin From cf43556515febb70123e0aa7a1ffab7c785b45c4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 10:19:33 -0400 Subject: [PATCH 26/67] Setup Script - Add Rules --- so-setup-network.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index ee6b9f022..7597ea5e7 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1312,6 +1312,7 @@ if (whiptail_you_sure); then LSPIPELINEBATCH=125 LSINPUTTHREADS=1 LSINPUTBATCHCOUNT=125 + RULESETUP=ETOPEN whiptail_make_changes get_main_ip # Add the user so we can sit back and relax From f22454b3741c9329eb24752703a9270d1fb895f2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 10:21:33 -0400 Subject: [PATCH 27/67] Setup Script - Remove static directory --- pillar/static/example.sls | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 pillar/static/example.sls diff --git a/pillar/static/example.sls b/pillar/static/example.sls deleted file mode 100644 index 2f1fdb178..000000000 --- a/pillar/static/example.sls +++ /dev/null @@ -1,6 +0,0 @@ -# This is for global salt items such as ntp servers etc. -static: - ntpserver: - homenet: - proxy: - masterupdate: From f9439fb43fe81bdedb0faf8328996d19e0a594f4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 10:28:40 -0400 Subject: [PATCH 28/67] Setup Script - --- so-setup-network.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index 7597ea5e7..ec0816929 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1302,6 +1302,9 @@ if (whiptail_you_sure); then whiptail_nids whiptail_bro_version + # Snag the HOME_NET + whiptail_homenet_master + whiptail_sensor_config # Set a bunch of stuff since this is eval es_heapsize From 7b9440dd709549c7ec1f338e32fe9f0b00f03fe7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 10:51:56 -0400 Subject: [PATCH 29/67] Elastic Search Module - Add eval support --- salt/elasticsearch/init.sls | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 857cea7d2..c1503f534 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -20,6 +20,14 @@ {% set dstats = salt['pillar.get']('master:dstats', '0') %} {% set esalert = salt['pillar.get']('master:elastalert', '1') %} +{% elif grains['role'] == 'so-eval' %} + +{% set esclustername = salt['pillar.get']('master:esclustername', '') %} +{% set esheap = salt['pillar.get']('master:esheap', '') %} +{% set freq = salt['pillar.get']('master:freq', '0') %} +{% set dstats = salt['pillar.get']('master:dstats', '0') %} +{% set esalert = salt['pillar.get']('master:elastalert', '1') %} + {% elif grains['role'] == 'so-node' %} {% set esclustername = salt['pillar.get']('node:esclustername', '') %} From 8dff89d6b4faa5e1ee9b0fc59d1f64950459bb59 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 10:58:53 -0400 Subject: [PATCH 30/67] LogStash Module - Add eval support --- salt/logstash/init.sls | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index a0c5db388..84d72125e 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -32,6 +32,14 @@ {% set dstats = salt['pillar.get']('master:domainstats', '0') %} {% set nodetype = salt['grains.get']('role', '') %} +{% elif grains['role'] == 'so-eval' %} + +{% set lsheap = salt['pillar.get']('master:lsheap', '') %} +{% set lsaccessip = salt['pillar.get']('master:lsaccessip', '') %} +{% set freq = salt['pillar.get']('master:freq', '0') %} +{% set dstats = salt['pillar.get']('master:domainstats', '0') %} +{% set nodetype = salt['grains.get']('role', '') %} + {% endif %} # Create the logstash group From 008e30023604e62ee06e0ff5c4a2a3416a100a63 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 11:22:09 -0400 Subject: [PATCH 31/67] Common Module - Fix nginx config --- salt/common/nginx/nginx.conf.so-eval | 110 +++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 salt/common/nginx/nginx.conf.so-eval diff --git a/salt/common/nginx/nginx.conf.so-eval b/salt/common/nginx/nginx.conf.so-eval new file mode 100644 index 000000000..bc8dbf39b --- /dev/null +++ b/salt/common/nginx/nginx.conf.so-eval @@ -0,0 +1,110 @@ +{%- set masterip = salt['pillar.get']('master:mainip', '') %} +# For more information on configuration, see: +# * Official English Documentation: http://nginx.org/en/docs/ +# * Official Russian Documentation: http://nginx.org/ru/docs/ + +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Load modular configuration files from the /etc/nginx/conf.d directory. + # See http://nginx.org/en/docs/ngx_core_module.html#include + # for more information. + include /etc/nginx/conf.d/*.conf; + + #server { + # listen 80 default_server; + # listen [::]:80 default_server; + # server_name _; + # root /opt/socore/html; + # index index.html; + + # Load configuration files for the default server block. + #include /etc/nginx/default.d/*.conf; + + # location / { + # } + + # error_page 404 /404.html; + # location = /40x.html { + # } + + # error_page 500 502 503 504 /50x.html; + # location = /50x.html { + # } + #} + server { + listen 80 default_server; + server_name _; + return 301 https://$host$request_uri; + } + + +# Settings for a TLS enabled server. + + server { + listen 443 ssl http2 default_server; + #listen [::]:443 ssl http2 default_server; + server_name _; + root /opt/socore/html; + index index.html; + + ssl_certificate "/etc/pki/nginx/server.crt"; + ssl_certificate_key "/etc/pki/nginx/server.key"; + ssl_session_cache shared:SSL:1m; + ssl_session_timeout 10m; + ssl_ciphers HIGH:!aNULL:!MD5; + ssl_prefer_server_ciphers on; + + # Load configuration files for the default server block. + #include /etc/nginx/default.d/*.conf; + + #location / { + # try_files $uri $uri.html /index.html; + # } + + location / { + proxy_pass http://{{ masterip }}:5601/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + error_page 404 /404.html; + location = /40x.html { + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + } + } + +} From a90e095687128dab687f3271d0550cb1d9b4485a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 11:24:58 -0400 Subject: [PATCH 32/67] LogStash Module - Fix Eval Logstash config --- salt/logstash/conf/conf.enabled.txt.so-eval | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 salt/logstash/conf/conf.enabled.txt.so-eval diff --git a/salt/logstash/conf/conf.enabled.txt.so-eval b/salt/logstash/conf/conf.enabled.txt.so-eval new file mode 100644 index 000000000..c33e46abe --- /dev/null +++ b/salt/logstash/conf/conf.enabled.txt.so-eval @@ -0,0 +1,17 @@ +# This is where can specify which LogStash configs get loaded. +# +# The custom folder on the master gets automatically synced to each logstash +# node. +# +# To enable a custom configuration see the following example and uncomment: +# /usr/share/logstash/pipeline.custom/1234_input_custom.conf +## +# All of the defaults are loaded. +/usr/share/logstash/pipeline.so/0000_input_syslogng.conf +/usr/share/logstash/pipeline.so/0001_input_json.conf +/usr/share/logstash/pipeline.so/0002_input_windows_json.conf +/usr/share/logstash/pipeline.so/0003_input_syslog.conf +/usr/share/logstash/pipeline.so/0005_input_suricata.conf +/usr/share/logstash/pipeline.dynamic/0006_input_beats.conf +/usr/share/logstash/pipeline.so/0007_input_import.conf +/usr/share/logstash/pipeline.dynamic/9999_output_redis.conf From b42e25d2502b99e3c36fa183571e80f25d762d1e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 11:25:36 -0400 Subject: [PATCH 33/67] LogStash Module - Fix Eval Logstash config --- salt/logstash/conf/conf.enabled.txt.so-eval | 94 ++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/salt/logstash/conf/conf.enabled.txt.so-eval b/salt/logstash/conf/conf.enabled.txt.so-eval index c33e46abe..71e50525f 100644 --- a/salt/logstash/conf/conf.enabled.txt.so-eval +++ b/salt/logstash/conf/conf.enabled.txt.so-eval @@ -14,4 +14,96 @@ /usr/share/logstash/pipeline.so/0005_input_suricata.conf /usr/share/logstash/pipeline.dynamic/0006_input_beats.conf /usr/share/logstash/pipeline.so/0007_input_import.conf -/usr/share/logstash/pipeline.dynamic/9999_output_redis.conf +/usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf +/usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf +/usr/share/logstash/pipeline.so/1002_preprocess_json.conf +/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf +/usr/share/logstash/pipeline.so/1004_preprocess_syslog_types.conf +/usr/share/logstash/pipeline.so/1026_preprocess_dhcp.conf +/usr/share/logstash/pipeline.so/1029_preprocess_esxi.conf +/usr/share/logstash/pipeline.so/1030_preprocess_greensql.conf +/usr/share/logstash/pipeline.so/1031_preprocess_iis.conf +/usr/share/logstash/pipeline.so/1032_preprocess_mcafee.conf +/usr/share/logstash/pipeline.so/1033_preprocess_snort.conf +/usr/share/logstash/pipeline.so/1034_preprocess_syslog.conf +/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf +/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf +/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf +/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf +/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf +/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf +/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf +/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf +/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf +/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf +/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf +/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf +/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf +/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf +/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf +/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf +/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf +/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf +/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf +/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf +/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf +/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf +/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf +/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf +/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf +/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf +/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf +/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf +/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf +/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf +/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf +/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf +/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf +/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf +/usr/share/logstash/pipeline.so/1998_test_data.conf +/usr/share/logstash/pipeline.so/2000_network_flow.conf +/usr/share/logstash/pipeline.so/6000_bro.conf +/usr/share/logstash/pipeline.so/6001_bro_import.conf +/usr/share/logstash/pipeline.so/6002_syslog.conf +/usr/share/logstash/pipeline.so/6101_switch_brocade.conf +/usr/share/logstash/pipeline.so/6200_firewall_fortinet.conf +/usr/share/logstash/pipeline.so/6201_firewall_pfsense.conf +/usr/share/logstash/pipeline.so/6300_windows.conf +/usr/share/logstash/pipeline.so/6301_dns_windows.conf +/usr/share/logstash/pipeline.so/6400_suricata.conf +/usr/share/logstash/pipeline.so/6500_ossec.conf +/usr/share/logstash/pipeline.so/6501_ossec_sysmon.conf +/usr/share/logstash/pipeline.so/6502_ossec_autoruns.conf +/usr/share/logstash/pipeline.so/6600_winlogbeat_sysmon.conf +/usr/share/logstash/pipeline.so/6700_winlogbeat.conf +/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf +/usr/share/logstash/pipeline.so/8001_postprocess_common_ip_augmentation.conf +#/usr/share/logstash/pipeline.so/8006_postprocess_dns.conf +#/usr/share/logstash/pipeline.so/8007_postprocess_dns_top1m_tagging.conf +/usr/share/logstash/pipeline.so/8007_postprocess_http.conf +#/usr/share/logstash/pipeline.so/8008_postprocess_dns_whois_age.conf +/usr/share/logstash/pipeline.so/8200_postprocess_tagging.conf +#/usr/share/logstash/pipeline.so/8502_postprocess_freq_analysis_bro_dns.conf +#/usr/share/logstash/pipeline.so/8503_postprocess_freq_analysis_bro_http.conf +#/usr/share/logstash/pipeline.so/8504_postprocess_freq_analysis_bro_ssl.conf +#/usr/share/logstash/pipeline.so/8505_postprocess_freq_analysis_bro_x509.conf +/usr/share/logstash/pipeline.so/8998_postprocess_log_elapsed.conf +/usr/share/logstash/pipeline.so/8999_postprocess_rename_type.conf +/usr/share/logstash/pipeline.dynamic/9000_output_bro.conf +/usr/share/logstash/pipeline.dynamic/9001_output_switch.conf +/usr/share/logstash/pipeline.dynamic/9002_output_import.conf +/usr/share/logstash/pipeline.dynamic/9004_output_flow.conf +/usr/share/logstash/pipeline.dynamic/9026_output_dhcp.conf +/usr/share/logstash/pipeline.dynamic/9029_output_esxi.conf +/usr/share/logstash/pipeline.dynamic/9030_output_greensql.conf +/usr/share/logstash/pipeline.dynamic/9031_output_iis.conf +/usr/share/logstash/pipeline.dynamic/9032_output_mcafee.conf +/usr/share/logstash/pipeline.dynamic/9033_output_snort.conf +/usr/share/logstash/pipeline.dynamic/9034_output_syslog.conf +/usr/share/logstash/pipeline.dynamic/9200_output_firewall.conf +/usr/share/logstash/pipeline.dynamic/9300_output_windows.conf +/usr/share/logstash/pipeline.dynamic/9301_output_dns_windows.conf +/usr/share/logstash/pipeline.dynamic/9400_output_suricata.conf +/usr/share/logstash/pipeline.dynamic/9500_output_beats.conf +/usr/share/logstash/pipeline.dynamic/9600_output_ossec.conf +/usr/share/logstash/pipeline.dynamic/9998_output_test_data.conf From 26272561d2a878702189c561674e44e8b4c13d31 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 11:41:48 -0400 Subject: [PATCH 34/67] Bro Module - Fix default interface --- salt/suricata/files/suricata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/suricata/files/suricata.yaml b/salt/suricata/files/suricata.yaml index fb3996b96..5a6c3ed72 100644 --- a/salt/suricata/files/suricata.yaml +++ b/salt/suricata/files/suricata.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -{%- set interface = salt['pillar.get']('sensor:interface', '') %} +{%- set interface = salt['pillar.get']('sensor:interface', 'bond0') %} {%- if salt['pillar.get']('sensor:homenet') %} {%- set homenet = salt['pillar.get']('sensor:hnsensor', '') %} {%- else %} From bdd946ae75b8fb4b784d7d5f261106ccda2e2a29 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 11:42:05 -0400 Subject: [PATCH 35/67] Bro Module - Fix default interface --- salt/bro/files/node.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/bro/files/node.cfg b/salt/bro/files/node.cfg index 2de9352ad..b886bc2cf 100644 --- a/salt/bro/files/node.cfg +++ b/salt/bro/files/node.cfg @@ -1,4 +1,4 @@ -{%- set interface = salt['pillar.get']('sensor:interface', '') %} +{%- set interface = salt['pillar.get']('sensor:interface', 'bond0') %} {%- if salt['pillar.get']('sensor:bro_pins') or salt['pillar.get']('sensor:bro_lbprocs') %} {%- if salt['pillar.get']('sensor:bro_proxies') %} From d6d47010f548712cfae065b429138c2fcb05db3c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 11:58:42 -0400 Subject: [PATCH 36/67] Load a pillar for eval mode --- pillar/top.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pillar/top.sls b/pillar/top.sls index 6095ef621..5fc5c69eb 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -13,6 +13,13 @@ base: - firewall.* - data.* + 'G@role:so-eval': + - masters.schedule + - masters.{{ grains.host }} + - static + - firewall.* + - data.* + 'G@role:so-node': - nodes.schedule - nodes.{{ grains.host }} From e31ad091afe5fd1a11c07969fef7aa6020e9a934 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 12:13:41 -0400 Subject: [PATCH 37/67] Setup Script - Fix double pillar opject --- so-setup-network.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index ec0816929..b08e50cd7 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -387,7 +387,6 @@ master_pillar() { if [ $INSTALLTYPE == 'EVALMODE' ]; then echo " freq: 1" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " domainstats: 1" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls - echo " ls_pipeline_workers: $CPUCORES" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " ls_pipeline_batch_size: 125" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " ls_input_threads: 1" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " ls_batch_count: 125" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls From 2bf1591208674cfdf43349c70815041e5c1bc4ff Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 14:22:58 -0400 Subject: [PATCH 38/67] Logstash Module - Fix Output --- salt/logstash/files/dynamic/9000_output_bro.conf | 4 +++- salt/logstash/files/dynamic/9001_output_switch.conf | 4 +++- salt/logstash/files/dynamic/9002_output_import.conf | 4 +++- salt/logstash/files/dynamic/9004_output_flow.conf | 4 +++- salt/logstash/files/dynamic/9026_output_dhcp.conf | 4 +++- salt/logstash/files/dynamic/9029_output_esxi.conf | 4 +++- salt/logstash/files/dynamic/9030_output_greensql.conf | 4 +++- salt/logstash/files/dynamic/9031_output_iis.conf | 4 +++- salt/logstash/files/dynamic/9032_output_mcafee.conf | 4 +++- salt/logstash/files/dynamic/9033_output_snort.conf | 4 +++- salt/logstash/files/dynamic/9034_output_syslog.conf | 4 +++- salt/logstash/files/dynamic/9200_output_firewall.conf | 4 +++- salt/logstash/files/dynamic/9300_output_windows.conf | 4 +++- salt/logstash/files/dynamic/9301_output_dns_windows.conf | 4 +++- salt/logstash/files/dynamic/9400_output_suricata.conf | 4 +++- salt/logstash/files/dynamic/9500_output_beats.conf | 4 +++- salt/logstash/files/dynamic/9600_output_ossec.conf | 4 +++- salt/logstash/files/dynamic/9998_output_test_data.conf | 4 +++- 18 files changed, 54 insertions(+), 18 deletions(-) diff --git a/salt/logstash/files/dynamic/9000_output_bro.conf b/salt/logstash/files/dynamic/9000_output_bro.conf index c4119e5de..01853270d 100644 --- a/salt/logstash/files/dynamic/9000_output_bro.conf +++ b/salt/logstash/files/dynamic/9000_output_bro.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics diff --git a/salt/logstash/files/dynamic/9001_output_switch.conf b/salt/logstash/files/dynamic/9001_output_switch.conf index c3dea84da..86ffbcec6 100644 --- a/salt/logstash/files/dynamic/9001_output_switch.conf +++ b/salt/logstash/files/dynamic/9001_output_switch.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics diff --git a/salt/logstash/files/dynamic/9002_output_import.conf b/salt/logstash/files/dynamic/9002_output_import.conf index 0a9d34726..80ab621aa 100644 --- a/salt/logstash/files/dynamic/9002_output_import.conf +++ b/salt/logstash/files/dynamic/9002_output_import.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Updated by: Doug Burks # Last Update: 5/16/2017 diff --git a/salt/logstash/files/dynamic/9004_output_flow.conf b/salt/logstash/files/dynamic/9004_output_flow.conf index ae37961c5..5ef3ca63e 100644 --- a/salt/logstash/files/dynamic/9004_output_flow.conf +++ b/salt/logstash/files/dynamic/9004_output_flow.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics diff --git a/salt/logstash/files/dynamic/9026_output_dhcp.conf b/salt/logstash/files/dynamic/9026_output_dhcp.conf index a6bb24850..0bc0b7233 100644 --- a/salt/logstash/files/dynamic/9026_output_dhcp.conf +++ b/salt/logstash/files/dynamic/9026_output_dhcp.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics diff --git a/salt/logstash/files/dynamic/9029_output_esxi.conf b/salt/logstash/files/dynamic/9029_output_esxi.conf index d7b37f03e..d0b9aedcb 100644 --- a/salt/logstash/files/dynamic/9029_output_esxi.conf +++ b/salt/logstash/files/dynamic/9029_output_esxi.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics diff --git a/salt/logstash/files/dynamic/9030_output_greensql.conf b/salt/logstash/files/dynamic/9030_output_greensql.conf index b8a4ff1ac..87d73a2ce 100644 --- a/salt/logstash/files/dynamic/9030_output_greensql.conf +++ b/salt/logstash/files/dynamic/9030_output_greensql.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics diff --git a/salt/logstash/files/dynamic/9031_output_iis.conf b/salt/logstash/files/dynamic/9031_output_iis.conf index 0073a18aa..7e03de66a 100644 --- a/salt/logstash/files/dynamic/9031_output_iis.conf +++ b/salt/logstash/files/dynamic/9031_output_iis.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics diff --git a/salt/logstash/files/dynamic/9032_output_mcafee.conf b/salt/logstash/files/dynamic/9032_output_mcafee.conf index efaa9ed24..154e7ffc2 100644 --- a/salt/logstash/files/dynamic/9032_output_mcafee.conf +++ b/salt/logstash/files/dynamic/9032_output_mcafee.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics diff --git a/salt/logstash/files/dynamic/9033_output_snort.conf b/salt/logstash/files/dynamic/9033_output_snort.conf index a16219494..36edace12 100644 --- a/salt/logstash/files/dynamic/9033_output_snort.conf +++ b/salt/logstash/files/dynamic/9033_output_snort.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics diff --git a/salt/logstash/files/dynamic/9034_output_syslog.conf b/salt/logstash/files/dynamic/9034_output_syslog.conf index 91a99d9b0..004373119 100644 --- a/salt/logstash/files/dynamic/9034_output_syslog.conf +++ b/salt/logstash/files/dynamic/9034_output_syslog.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics diff --git a/salt/logstash/files/dynamic/9200_output_firewall.conf b/salt/logstash/files/dynamic/9200_output_firewall.conf index 3e9f658a6..203372de1 100644 --- a/salt/logstash/files/dynamic/9200_output_firewall.conf +++ b/salt/logstash/files/dynamic/9200_output_firewall.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics diff --git a/salt/logstash/files/dynamic/9300_output_windows.conf b/salt/logstash/files/dynamic/9300_output_windows.conf index a0a1e12c7..89aa6f724 100644 --- a/salt/logstash/files/dynamic/9300_output_windows.conf +++ b/salt/logstash/files/dynamic/9300_output_windows.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics diff --git a/salt/logstash/files/dynamic/9301_output_dns_windows.conf b/salt/logstash/files/dynamic/9301_output_dns_windows.conf index 871a479b1..d8857ee3b 100644 --- a/salt/logstash/files/dynamic/9301_output_dns_windows.conf +++ b/salt/logstash/files/dynamic/9301_output_dns_windows.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics diff --git a/salt/logstash/files/dynamic/9400_output_suricata.conf b/salt/logstash/files/dynamic/9400_output_suricata.conf index 41771e41c..f5846ab00 100644 --- a/salt/logstash/files/dynamic/9400_output_suricata.conf +++ b/salt/logstash/files/dynamic/9400_output_suricata.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics diff --git a/salt/logstash/files/dynamic/9500_output_beats.conf b/salt/logstash/files/dynamic/9500_output_beats.conf index 641df21c6..50952441d 100644 --- a/salt/logstash/files/dynamic/9500_output_beats.conf +++ b/salt/logstash/files/dynamic/9500_output_beats.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Wes Lambert # Last Update: 09/14/2018 diff --git a/salt/logstash/files/dynamic/9600_output_ossec.conf b/salt/logstash/files/dynamic/9600_output_ossec.conf index ea1180ae5..23a8af16b 100644 --- a/salt/logstash/files/dynamic/9600_output_ossec.conf +++ b/salt/logstash/files/dynamic/9600_output_ossec.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics diff --git a/salt/logstash/files/dynamic/9998_output_test_data.conf b/salt/logstash/files/dynamic/9998_output_test_data.conf index 87b59db00..225ede01d 100644 --- a/salt/logstash/files/dynamic/9998_output_test_data.conf +++ b/salt/logstash/files/dynamic/9998_output_test_data.conf @@ -1,4 +1,6 @@ -{%- if grains['role'] != 'so-master' -%} +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics From b53105fa73f36e7ee7aebd9648634d9ff3240d6d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 14:24:47 -0400 Subject: [PATCH 39/67] Enable Filebeat --- salt/top.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/top.sls b/salt/top.sls index ba7ab520f..41b0b1e20 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -23,6 +23,7 @@ base: - pcap - suricata - bro + - filebeat 'G@role:so-master': From 7d08f32487883445c4f78bf5292ec82487119419 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 14:48:20 -0400 Subject: [PATCH 40/67] SSL Module - Fix Filebeat Cert --- salt/ssl/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 27a08d276..4b12a75ac 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -78,7 +78,7 @@ fbcrtlink: backup: True {% endif %} -{% if grains['role'] == 'so-SENSOR' %} +{% if grains['role'] == 'so-SENSOR' or grains['role'] == 'so-eval' %} fbcertdir: file.directory: From 8f9450625acc17b58d0851e8889e9b7aadee69d5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 15:05:00 -0400 Subject: [PATCH 41/67] Firewall Module - Firewall rules for eval --- so-setup-network.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index b08e50cd7..6303c77ec 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1340,6 +1340,7 @@ if (whiptail_you_sure); then node_pillar salt_firstcheckin accept_salt_key_local + set_initial_firewall_policy salt_checkin_message salt_checkin checkin_at_boot From 0268f98f8ea74d393bdf0e30d05b26d39e5eb899 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 16:53:23 -0400 Subject: [PATCH 42/67] Fix connectivity --- salt/elasticsearch/files/elasticsearch.yml | 4 ++-- salt/firewall/init.sls | 2 +- so-setup-network.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index bc3ad3aee..b90de2e14 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -1,4 +1,4 @@ -{% if grains['role'] == 'so-master' %} +{% if grains['role'] == 'so-SENSOR' or grains['role'] == 'so-eval' %} {%- set esclustername = salt['pillar.get']('master:esclustername', '') %} cluster.name: "{{ esclustername }}" network.host: 0.0.0.0 @@ -19,6 +19,6 @@ discovery.zen.minimum_master_nodes: 1 path.logs: /var/log/elasticsearch action.destructive_requires_name: true transport.bind_host: 0.0.0.0 -transport.publish_host: {{ nodeip }} +transport.publish_host: {{ nodeip }} transport.publish_port: 9300 {%- endif %} diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index 6b0a3737e..b79a57f31 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -87,7 +87,7 @@ enable_docker_user_established: - ctstate: 'RELATED,ESTABLISHED' # Rules if you are a Master -{% if grains['role'] == 'so-master' %} +{% if grains['role'] == 'so-SENSOR' or grains['role'] == 'so-eval' %} #This should be more granular iptables_allow_master_docker: iptables.insert: diff --git a/so-setup-network.sh b/so-setup-network.sh index 6303c77ec..d348bb839 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1338,9 +1338,9 @@ if (whiptail_you_sure); then configure_minion eval set_node_type node_pillar + set_initial_firewall_policy salt_firstcheckin accept_salt_key_local - set_initial_firewall_policy salt_checkin_message salt_checkin checkin_at_boot From d15ae474c6e8430efb887c0d66892102cb13a0b8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 19:55:21 -0400 Subject: [PATCH 43/67] Try and Fix certs --- pillar/top.sls | 1 + salt/filebeat/init.sls | 1 + 2 files changed, 2 insertions(+) diff --git a/pillar/top.sls b/pillar/top.sls index 5fc5c69eb..064d2cff3 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -19,6 +19,7 @@ base: - static - firewall.* - data.* + - brologs 'G@role:so-node': - nodes.schedule diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 6fb65bd63..1523f7a13 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -63,4 +63,5 @@ so-filebeat: - /opt/so/log/suricata:/suricata:ro - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro + - /opt/so/conf/filebeat/etc/pki/filebeat.p8:/usr/share/filebeat/filebeat.p8:ro - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro From 305c80b115aeaa2807e7cc9b368ee1f8830699b3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Oct 2018 20:00:55 -0400 Subject: [PATCH 44/67] Try and Fix certs --- salt/filebeat/init.sls | 1 - salt/ssl/init.sls | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 1523f7a13..6fb65bd63 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -63,5 +63,4 @@ so-filebeat: - /opt/so/log/suricata:/suricata:ro - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro - - /opt/so/conf/filebeat/etc/pki/filebeat.p8:/usr/share/filebeat/filebeat.p8:ro - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 4b12a75ac..c403154fb 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -16,7 +16,8 @@ m2cryptopkgs: - python-m2crypto {% endif %} -{% if grains['role'] == 'so-master' %} +{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %} + # Request a cert and drop it where it needs to go to be distributed /etc/pki/filebeat.crt: x509.certificate_managed: From 39677afa74e651310b9a66f8fc79ff0e7674119d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 1 Nov 2018 17:41:11 -0400 Subject: [PATCH 45/67] Logstash Module - Add new logstash config for eval --- salt/logstash/conf/conf.enabled.txt.so-eval | 2 +- salt/top.sls | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/logstash/conf/conf.enabled.txt.so-eval b/salt/logstash/conf/conf.enabled.txt.so-eval index 71e50525f..e5ce9c803 100644 --- a/salt/logstash/conf/conf.enabled.txt.so-eval +++ b/salt/logstash/conf/conf.enabled.txt.so-eval @@ -12,8 +12,8 @@ /usr/share/logstash/pipeline.so/0002_input_windows_json.conf /usr/share/logstash/pipeline.so/0003_input_syslog.conf /usr/share/logstash/pipeline.so/0005_input_suricata.conf -/usr/share/logstash/pipeline.dynamic/0006_input_beats.conf /usr/share/logstash/pipeline.so/0007_input_import.conf +/usr/share/logstash/pipeline.so/0008_input_eval.conf /usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf /usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf /usr/share/logstash/pipeline.so/1002_preprocess_json.conf diff --git a/salt/top.sls b/salt/top.sls index 41b0b1e20..ba7ab520f 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -23,7 +23,6 @@ base: - pcap - suricata - bro - - filebeat 'G@role:so-master': From f2bfef58c8efb623b5f0f7fcb86d7202a5d6ef7a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 1 Nov 2018 18:03:05 -0400 Subject: [PATCH 46/67] Logstash Module - Fix ES output logic --- salt/logstash/files/dynamic/9000_output_bro.conf | 2 +- salt/logstash/files/dynamic/9001_output_switch.conf | 2 +- salt/logstash/files/dynamic/9002_output_import.conf | 2 +- salt/logstash/files/dynamic/9004_output_flow.conf | 2 +- salt/logstash/files/dynamic/9026_output_dhcp.conf | 2 +- salt/logstash/files/dynamic/9029_output_esxi.conf | 2 +- salt/logstash/files/dynamic/9030_output_greensql.conf | 2 +- salt/logstash/files/dynamic/9031_output_iis.conf | 2 +- salt/logstash/files/dynamic/9032_output_mcafee.conf | 2 +- salt/logstash/files/dynamic/9033_output_snort.conf | 2 +- salt/logstash/files/dynamic/9034_output_syslog.conf | 2 +- salt/logstash/files/dynamic/9200_output_firewall.conf | 2 +- salt/logstash/files/dynamic/9300_output_windows.conf | 2 +- salt/logstash/files/dynamic/9301_output_dns_windows.conf | 2 +- salt/logstash/files/dynamic/9400_output_suricata.conf | 2 +- salt/logstash/files/dynamic/9500_output_beats.conf | 2 +- salt/logstash/files/dynamic/9600_output_ossec.conf | 2 +- salt/logstash/files/dynamic/9998_output_test_data.conf | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/salt/logstash/files/dynamic/9000_output_bro.conf b/salt/logstash/files/dynamic/9000_output_bro.conf index 01853270d..54d2d20ca 100644 --- a/salt/logstash/files/dynamic/9000_output_bro.conf +++ b/salt/logstash/files/dynamic/9000_output_bro.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -27,4 +28,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9001_output_switch.conf b/salt/logstash/files/dynamic/9001_output_switch.conf index 86ffbcec6..949a738ab 100644 --- a/salt/logstash/files/dynamic/9001_output_switch.conf +++ b/salt/logstash/files/dynamic/9001_output_switch.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -24,4 +25,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9002_output_import.conf b/salt/logstash/files/dynamic/9002_output_import.conf index 80ab621aa..88fbc7551 100644 --- a/salt/logstash/files/dynamic/9002_output_import.conf +++ b/salt/logstash/files/dynamic/9002_output_import.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Updated by: Doug Burks # Last Update: 5/16/2017 @@ -24,4 +25,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9004_output_flow.conf b/salt/logstash/files/dynamic/9004_output_flow.conf index 5ef3ca63e..3dbd34f16 100644 --- a/salt/logstash/files/dynamic/9004_output_flow.conf +++ b/salt/logstash/files/dynamic/9004_output_flow.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -24,4 +25,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9026_output_dhcp.conf b/salt/logstash/files/dynamic/9026_output_dhcp.conf index 0bc0b7233..a63ac5f98 100644 --- a/salt/logstash/files/dynamic/9026_output_dhcp.conf +++ b/salt/logstash/files/dynamic/9026_output_dhcp.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -23,4 +24,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9029_output_esxi.conf b/salt/logstash/files/dynamic/9029_output_esxi.conf index d0b9aedcb..229de6b9c 100644 --- a/salt/logstash/files/dynamic/9029_output_esxi.conf +++ b/salt/logstash/files/dynamic/9029_output_esxi.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -22,4 +23,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9030_output_greensql.conf b/salt/logstash/files/dynamic/9030_output_greensql.conf index 87d73a2ce..a6d16b95d 100644 --- a/salt/logstash/files/dynamic/9030_output_greensql.conf +++ b/salt/logstash/files/dynamic/9030_output_greensql.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -22,4 +23,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9031_output_iis.conf b/salt/logstash/files/dynamic/9031_output_iis.conf index 7e03de66a..6650d8a7d 100644 --- a/salt/logstash/files/dynamic/9031_output_iis.conf +++ b/salt/logstash/files/dynamic/9031_output_iis.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -23,4 +24,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9032_output_mcafee.conf b/salt/logstash/files/dynamic/9032_output_mcafee.conf index 154e7ffc2..ca982967d 100644 --- a/salt/logstash/files/dynamic/9032_output_mcafee.conf +++ b/salt/logstash/files/dynamic/9032_output_mcafee.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -23,4 +24,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9033_output_snort.conf b/salt/logstash/files/dynamic/9033_output_snort.conf index 36edace12..a953a2db2 100644 --- a/salt/logstash/files/dynamic/9033_output_snort.conf +++ b/salt/logstash/files/dynamic/9033_output_snort.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -26,4 +27,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9034_output_syslog.conf b/salt/logstash/files/dynamic/9034_output_syslog.conf index 004373119..56a6527b8 100644 --- a/salt/logstash/files/dynamic/9034_output_syslog.conf +++ b/salt/logstash/files/dynamic/9034_output_syslog.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Updated by: Doug Burks @@ -25,4 +26,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9200_output_firewall.conf b/salt/logstash/files/dynamic/9200_output_firewall.conf index 203372de1..b2ad43963 100644 --- a/salt/logstash/files/dynamic/9200_output_firewall.conf +++ b/salt/logstash/files/dynamic/9200_output_firewall.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -26,4 +27,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9300_output_windows.conf b/salt/logstash/files/dynamic/9300_output_windows.conf index 89aa6f724..d3f9d1919 100644 --- a/salt/logstash/files/dynamic/9300_output_windows.conf +++ b/salt/logstash/files/dynamic/9300_output_windows.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -24,4 +25,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9301_output_dns_windows.conf b/salt/logstash/files/dynamic/9301_output_dns_windows.conf index d8857ee3b..8a56b7044 100644 --- a/salt/logstash/files/dynamic/9301_output_dns_windows.conf +++ b/salt/logstash/files/dynamic/9301_output_dns_windows.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -24,4 +25,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9400_output_suricata.conf b/salt/logstash/files/dynamic/9400_output_suricata.conf index f5846ab00..4bffd7f0a 100644 --- a/salt/logstash/files/dynamic/9400_output_suricata.conf +++ b/salt/logstash/files/dynamic/9400_output_suricata.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -24,4 +25,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9500_output_beats.conf b/salt/logstash/files/dynamic/9500_output_beats.conf index 50952441d..30900cb93 100644 --- a/salt/logstash/files/dynamic/9500_output_beats.conf +++ b/salt/logstash/files/dynamic/9500_output_beats.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Wes Lambert # Last Update: 09/14/2018 filter { @@ -22,4 +23,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9600_output_ossec.conf b/salt/logstash/files/dynamic/9600_output_ossec.conf index 23a8af16b..71d0c28aa 100644 --- a/salt/logstash/files/dynamic/9600_output_ossec.conf +++ b/salt/logstash/files/dynamic/9600_output_ossec.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Updated by: Doug Burks @@ -26,4 +27,3 @@ output { } } } -{%- endif %} diff --git a/salt/logstash/files/dynamic/9998_output_test_data.conf b/salt/logstash/files/dynamic/9998_output_test_data.conf index 225ede01d..4e83aa185 100644 --- a/salt/logstash/files/dynamic/9998_output_test_data.conf +++ b/salt/logstash/files/dynamic/9998_output_test_data.conf @@ -2,6 +2,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -23,4 +24,3 @@ output { } } } -{%- endif %} From 01db0e6d61dee034c865f84449dcdd7a186667c6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 1 Nov 2018 18:49:04 -0400 Subject: [PATCH 47/67] Logstash Module - Add path to logs --- salt/logstash/init.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 84d72125e..c597cf5fd 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -141,3 +141,7 @@ so-logstash: - /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro + {%- if grains['role'] == 'so-eval' %} + - /nsm/bro/logs:/nsm/bro/logs:ro + - /opt/so/log/suricata:/suricata:ro + {%- endif %} From 56a9dee93878413778ef2837a3189c8be30161e2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 1 Nov 2018 19:11:50 -0400 Subject: [PATCH 48/67] Logstash Module - Add path to logs --- salt/logstash/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index c597cf5fd..929529c70 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -142,6 +142,6 @@ so-logstash: - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro {%- if grains['role'] == 'so-eval' %} - - /nsm/bro/logs:/nsm/bro/logs:ro + - /nsm/bro:/nsm/bro:ro - /opt/so/log/suricata:/suricata:ro {%- endif %} From 74ca8450d18723a2708a7d274ffe11e7747d5ce3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 09:43:53 -0400 Subject: [PATCH 49/67] Utility Module - Fix the search for eval --- salt/top.sls | 2 +- salt/utility/bin/eval.sh | 30 ++++++++++++++++++++++++++++++ salt/utility/init.sls | 8 ++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 salt/utility/bin/eval.sh diff --git a/salt/top.sls b/salt/top.sls index ba7ab520f..23878e70e 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -19,10 +19,10 @@ base: - elasticsearch - logstash - kibana - - utility - pcap - suricata - bro + - utility 'G@role:so-master': diff --git a/salt/utility/bin/eval.sh b/salt/utility/bin/eval.sh new file mode 100644 index 000000000..effbdfd33 --- /dev/null +++ b/salt/utility/bin/eval.sh @@ -0,0 +1,30 @@ +{%- set ES = salt['pillar.get']('master:mainip', '') -%} +{%- set MASTER = grains['master'] %} +# Wait for ElasticSearch to come up, so that we can query for version infromation +echo -n "Waiting for ElasticSearch..." +COUNT=0 +ELASTICSEARCH_CONNECTED="no" +while [[ "$COUNT" -le 30 ]]; do + curl --output /dev/null --silent --head --fail http://{{ ES }}:9200 + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi +done +if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo + + exit +fi + +echo "Applying cross cluster search config..." + curl -s -XPUT http://{{ ES }}:9200/_cluster/settings \ + -H 'Content-Type: application/json' \ + -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MASTER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" diff --git a/salt/utility/init.sls b/salt/utility/init.sls index 5cb628d50..845da59c7 100644 --- a/salt/utility/init.sls +++ b/salt/utility/init.sls @@ -11,3 +11,11 @@ crossclusterson: {% endif %} {% endif %} +{% if grains['role'] == 'so-eval' %} +fixsearch: + cmd.script: + - shell: /bin/bash + - runas: socore + - source: salt://utility/bin/eval.sh + - template: jinja +{% endif %} From 84df02ebe6c656673a23d97a65b3910cb6912b04 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 10:03:40 -0400 Subject: [PATCH 50/67] Bro Module - Turn off JSON --- salt/bro/files/local.bro | 2 +- salt/bro/files/local.bro.community | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/bro/files/local.bro b/salt/bro/files/local.bro index d97fbd6f3..dbad5cf61 100644 --- a/salt/bro/files/local.bro +++ b/salt/bro/files/local.bro @@ -141,4 +141,4 @@ #@load custom/somebropolicy.bro # Write logs in JSON -redef LogAscii::use_json = T; +#redef LogAscii::use_json = T; diff --git a/salt/bro/files/local.bro.community b/salt/bro/files/local.bro.community index 40476d8e7..58d079552 100644 --- a/salt/bro/files/local.bro.community +++ b/salt/bro/files/local.bro.community @@ -129,4 +129,4 @@ #@load custom/somebropolicy.bro # Use JSON -redef LogAscii::use_json = T; +#redef LogAscii::use_json = T; From 96f5a310948eea9ed028c5df0c3d9faee9564983 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 10:34:28 -0400 Subject: [PATCH 51/67] Setup Script - Remove some questions for eval mode --- so-setup-network.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index d348bb839..e69994af4 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1297,14 +1297,9 @@ if (whiptail_you_sure); then # Select which NICs are in the bond whiptail_bond_nics - # Set the NIDS to suricata - whiptail_nids - - whiptail_bro_version # Snag the HOME_NET whiptail_homenet_master - whiptail_sensor_config # Set a bunch of stuff since this is eval es_heapsize ls_heapsize @@ -1315,6 +1310,9 @@ if (whiptail_you_sure); then LSINPUTTHREADS=1 LSINPUTBATCHCOUNT=125 RULESETUP=ETOPEN + NSMSETUP=BASIC + NIDS=Suricata + BROVERSION=COMMUNITY whiptail_make_changes get_main_ip # Add the user so we can sit back and relax From 15c3ecc568414005ea5f6ed55e0c26228b81effc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 11:07:42 -0400 Subject: [PATCH 52/67] Setup Script - Remove some questions for eval mode --- so-setup-network.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index e69994af4..6665481cd 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1342,6 +1342,7 @@ if (whiptail_you_sure); then salt_checkin_message salt_checkin checkin_at_boot + whiptail_setup_complete fi ################### From 3cf99039f40ea39ec02f47716c347f5430d60a5c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 12:55:40 -0400 Subject: [PATCH 53/67] Setup Script - Enable salt on rebootd --- so-setup-network.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index 6665481cd..9a0e95645 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -513,10 +513,12 @@ saltify() { yum clean expire-cache yum -y install salt-minion yum-utils device-mapper-persistent-data lvm2 openssl yum -y update + systemctl enable salt-minion # Nasty hack but required for now if [ $INSTALLTYPE == 'MASTERONLY' ] || [ $INSTALLTYPE == 'EVALMODE' ]; then yum -y install salt-master python-m2crypto salt-minion m2crypto + systemctl enable salt-master else yum -y install salt-minion python-m2m2crypto m2crypto fi From 16172d894e9bb76d7f10a70134cacbc084f07d8d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 13:24:28 -0400 Subject: [PATCH 54/67] Logstash Module - Remove .eval --- salt/logstash/conf/conf.enabled.txt.eval | 106 ----------------------- 1 file changed, 106 deletions(-) delete mode 100644 salt/logstash/conf/conf.enabled.txt.eval diff --git a/salt/logstash/conf/conf.enabled.txt.eval b/salt/logstash/conf/conf.enabled.txt.eval deleted file mode 100644 index 404005b12..000000000 --- a/salt/logstash/conf/conf.enabled.txt.eval +++ /dev/null @@ -1,106 +0,0 @@ -# This is where can specify which LogStash configs get loaded. -# -# The custom folder on the master gets automatically synced to each logstash -# node. -# -# To enable a custom configuration see the following example and uncomment: -# /usr/share/logstash/pipeline.custom/1234_input_custom.conf -## -# All of the defaults are loaded. -/usr/share/logstash/pipeline.so/0000_input_syslogng.conf -/usr/share/logstash/pipeline.so/0001_input_json.conf -/usr/share/logstash/pipeline.so/0002_input_windows_json.conf -/usr/share/logstash/pipeline.so/0003_input_syslog.conf -/usr/share/logstash/pipeline.so/0005_input_suricata.conf -/usr/share/logstash/pipeline.so/0006_input_beats.conf -/usr/share/logstash/pipeline.so/0007_input_import.conf -/usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf -/usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf -/usr/share/logstash/pipeline.so/1002_preprocess_json.conf -/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf -/usr/share/logstash/pipeline.so/1004_preprocess_syslog_types.conf -/usr/share/logstash/pipeline.so/1026_preprocess_dhcp.conf -/usr/share/logstash/pipeline.so/1029_preprocess_esxi.conf -/usr/share/logstash/pipeline.so/1030_preprocess_greensql.conf -/usr/share/logstash/pipeline.so/1031_preprocess_iis.conf -/usr/share/logstash/pipeline.so/1032_preprocess_mcafee.conf -/usr/share/logstash/pipeline.so/1033_preprocess_snort.conf -/usr/share/logstash/pipeline.so/1034_preprocess_syslog.conf -/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf -/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf -/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf -/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf -/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf -/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf -/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf -/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf -/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf -/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf -/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf -/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf -/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf -/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf -/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf -/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf -/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf -/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf -/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf -/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf -/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf -/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf -/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf -/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf -/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf -/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf -/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf -/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf -/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf -/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf -/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf -/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf -/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf -/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf -/usr/share/logstash/pipeline.so/1998_test_data.conf -/usr/share/logstash/pipeline.so/2000_network_flow.conf -/usr/share/logstash/pipeline.so/6000_bro.conf -/usr/share/logstash/pipeline.so/6001_bro_import.conf -/usr/share/logstash/pipeline.so/6002_syslog.conf -/usr/share/logstash/pipeline.so/6101_switch_brocade.conf -/usr/share/logstash/pipeline.so/6200_firewall_fortinet.conf -/usr/share/logstash/pipeline.so/6201_firewall_pfsense.conf -/usr/share/logstash/pipeline.so/6300_windows.conf -/usr/share/logstash/pipeline.so/6301_dns_windows.conf -/usr/share/logstash/pipeline.so/6400_suricata.conf -/usr/share/logstash/pipeline.so/6500_ossec.conf -/usr/share/logstash/pipeline.so/6501_ossec_sysmon.conf -/usr/share/logstash/pipeline.so/6502_ossec_autoruns.conf -/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf -/usr/share/logstash/pipeline.so/8001_postprocess_common_ip_augmentation.conf -/usr/share/logstash/pipeline.so/8006_postprocess_dns.conf -/usr/share/logstash/pipeline.so/8007_postprocess_dns_top1m_tagging.conf -/usr/share/logstash/pipeline.so/8007_postprocess_http.conf -/usr/share/logstash/pipeline.so/8008_postprocess_dns_whois_age.conf -/usr/share/logstash/pipeline.so/8200_postprocess_tagging.conf -/usr/share/logstash/pipeline.so/8502_postprocess_freq_analysis_bro_dns.conf -/usr/share/logstash/pipeline.so/8503_postprocess_freq_analysis_bro_http.conf -/usr/share/logstash/pipeline.so/8504_postprocess_freq_analysis_bro_ssl.conf -/usr/share/logstash/pipeline.so/8505_postprocess_freq_analysis_bro_x509.conf -/usr/share/logstash/pipeline.so/8998_postprocess_log_elapsed.conf -/usr/share/logstash/pipeline.so/8999_postprocess_rename_type.conf -/usr/share/logstash/pipeline.so/9000_output_bro.conf -/usr/share/logstash/pipeline.so/9001_output_switch.conf -/usr/share/logstash/pipeline.so/9002_output_import.conf -/usr/share/logstash/pipeline.so/9004_output_flow.conf -/usr/share/logstash/pipeline.so/9026_output_dhcp.conf -/usr/share/logstash/pipeline.so/9029_output_esxi.conf -/usr/share/logstash/pipeline.so/9030_output_greensql.conf -/usr/share/logstash/pipeline.so/9031_output_iis.conf -/usr/share/logstash/pipeline.so/9032_output_mcafee.conf -/usr/share/logstash/pipeline.so/9033_output_snort.conf -/usr/share/logstash/pipeline.so/9034_output_syslog.conf -/usr/share/logstash/pipeline.so/9200_output_firewall.conf -/usr/share/logstash/pipeline.so/9300_output_windows.conf -/usr/share/logstash/pipeline.so/9301_output_dns_windows.conf -/usr/share/logstash/pipeline.so/9400_output_suricata.conf -/usr/share/logstash/pipeline.so/9500_output_beats.conf -/usr/share/logstash/pipeline.so/9998_output_test_data.conf From 8cd7278ad1f0fed424ac5328a75d4a9f5f671acb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 14:16:11 -0400 Subject: [PATCH 55/67] ElasticSearch Module - Fix logic for master --- salt/elasticsearch/files/elasticsearch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index b90de2e14..73f3c9239 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -1,4 +1,4 @@ -{% if grains['role'] == 'so-SENSOR' or grains['role'] == 'so-eval' %} +{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %} {%- set esclustername = salt['pillar.get']('master:esclustername', '') %} cluster.name: "{{ esclustername }}" network.host: 0.0.0.0 From 660ab40670a7baa07f4fc7252c5fddc035a5e9a7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 14:33:26 -0400 Subject: [PATCH 56/67] Firewall Module - Fix logic for master --- salt/firewall/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index b79a57f31..c3be1eaed 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -87,7 +87,7 @@ enable_docker_user_established: - ctstate: 'RELATED,ESTABLISHED' # Rules if you are a Master -{% if grains['role'] == 'so-SENSOR' or grains['role'] == 'so-eval' %} +{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %} #This should be more granular iptables_allow_master_docker: iptables.insert: From ccff173b019487e5c6c38af5391c762aec29a839 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 15:26:34 -0400 Subject: [PATCH 57/67] Update Readme --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 33d594fad..d28361812 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,30 @@ Installation: +If you are using CentOS 7 there are a couple pre-requisites: + +``` +sudo yum -y install bind-utils +sudo hostnamectl set-hostname YOURHOSTNAME +sudo reboot +``` +Once you resolve those requirements or are using Ubuntu do the following: + ``` git clone https://github.com/Security-Onion-Solutions/securityonion-saltstack cd securityonion-saltstack sudo bash so-setup-network.sh ``` +Allow Access to Kibana: + +For a single host: +``` +sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh analyst 192.168.30.1 +``` +For a network range: +``` +sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh analyst 192.168.30.0/24 +``` +Then connect to your master via https://YOURMASTER See the [FAQ](https://github.com/Security-Onion-Solutions/securityonion-saltstack/wiki/FAQ) on the Hybrid Hunter wiki. From 8b553c391999149f073636285841db5c1787ecd4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 15:28:04 -0400 Subject: [PATCH 58/67] Update Readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d28361812..c82c75349 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ sudo yum -y install bind-utils sudo hostnamectl set-hostname YOURHOSTNAME sudo reboot ``` -Once you resolve those requirements or are using Ubuntu do the following: +Once you resolve those requirements or are using Ubuntu 16.04 do the following: ``` git clone https://github.com/Security-Onion-Solutions/securityonion-saltstack From 7fad6ab1d6605c6b7c1ecffdf74557094fec7d67 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 15:29:55 -0400 Subject: [PATCH 59/67] Update Readme --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index c82c75349..c4830f1e9 100644 --- a/README.md +++ b/README.md @@ -29,3 +29,15 @@ sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh analyst 192.168.30.0/24 Then connect to your master via https://YOURMASTER See the [FAQ](https://github.com/Security-Onion-Solutions/securityonion-saltstack/wiki/FAQ) on the Hybrid Hunter wiki. + +**Warnings and Disclaimers** + +This technology PREVIEW is PRE-ALPHA, BLEEDING EDGE, and TOTALLY UNSUPPORTED! +If this breaks your system, you get to keep both pieces! +This script is a work in progress and is in constant flux. +This script is intended to build a quick prototype proof of concept so you can see what our ultimate ELK configuration might look like. This configuration will change drastically over time leading up to the final release. +Do NOT run this on a system that you care about! +Do NOT run this on a system that has data that you care about! +This script should only be run on a TEST box with TEST data! +This script is only designed for standalone boxes and does NOT support distributed deployments. +Use of this script may result in nausea, vomiting, or a burning sensation. From 99d5d8e2356580a79bd33eb62bf1d3c8293e9d3d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 15:30:43 -0400 Subject: [PATCH 60/67] Update Readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c4830f1e9..5912ace8f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Security Onion Hybrid Hunter Tech Preview +# Security Onion Hybrid Hunter Tech Preview 1.0.1 Installation: From fe2d9340437293de51dc460c4dcd9b11c7b51ddd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 15:31:59 -0400 Subject: [PATCH 61/67] Update Readme --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 5912ace8f..b917c8cf1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,18 @@ # Security Onion Hybrid Hunter Tech Preview 1.0.1 +**Warnings and Disclaimers** + +This technology PREVIEW is PRE-ALPHA, BLEEDING EDGE, and TOTALLY UNSUPPORTED! +If this breaks your system, you get to keep both pieces! +This script is a work in progress and is in constant flux. +This script is intended to build a quick prototype proof of concept so you can see what our ultimate ELK configuration might look like. This configuration will change drastically over time leading up to the final release. +Do NOT run this on a system that you care about! +Do NOT run this on a system that has data that you care about! +This script should only be run on a TEST box with TEST data! +This script is only designed for standalone boxes and does NOT support distributed deployments. +Use of this script may result in nausea, vomiting, or a burning sensation. + + Installation: If you are using CentOS 7 there are a couple pre-requisites: From be57387d04986adeb52fc6fc661aff08a8fe656a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 15:32:53 -0400 Subject: [PATCH 62/67] Update Readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b917c8cf1..ed1f2f7a9 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This technology PREVIEW is PRE-ALPHA, BLEEDING EDGE, and TOTALLY UNSUPPORTED! If this breaks your system, you get to keep both pieces! This script is a work in progress and is in constant flux. -This script is intended to build a quick prototype proof of concept so you can see what our ultimate ELK configuration might look like. This configuration will change drastically over time leading up to the final release. +This script is intended to build a quick prototype proof of concept so you can see what our new platform might look like. This configuration will change drastically over time leading up to the final release. Do NOT run this on a system that you care about! Do NOT run this on a system that has data that you care about! This script should only be run on a TEST box with TEST data! From 631e4316fde96ea760811103ce9eaa8179a414a0 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 15:33:41 -0400 Subject: [PATCH 63/67] Update Readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ed1f2f7a9..24281e47e 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ **Warnings and Disclaimers** +``` This technology PREVIEW is PRE-ALPHA, BLEEDING EDGE, and TOTALLY UNSUPPORTED! If this breaks your system, you get to keep both pieces! This script is a work in progress and is in constant flux. @@ -11,7 +12,7 @@ Do NOT run this on a system that has data that you care about! This script should only be run on a TEST box with TEST data! This script is only designed for standalone boxes and does NOT support distributed deployments. Use of this script may result in nausea, vomiting, or a burning sensation. - +``` Installation: From f61f56bda9f5a8394efd5862ace4689eeb23277d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 15:34:30 -0400 Subject: [PATCH 64/67] Update Readme --- README.md | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 24281e47e..dace6ef32 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,15 @@ **Warnings and Disclaimers** -``` -This technology PREVIEW is PRE-ALPHA, BLEEDING EDGE, and TOTALLY UNSUPPORTED! -If this breaks your system, you get to keep both pieces! -This script is a work in progress and is in constant flux. -This script is intended to build a quick prototype proof of concept so you can see what our new platform might look like. This configuration will change drastically over time leading up to the final release. -Do NOT run this on a system that you care about! -Do NOT run this on a system that has data that you care about! -This script should only be run on a TEST box with TEST data! -This script is only designed for standalone boxes and does NOT support distributed deployments. -Use of this script may result in nausea, vomiting, or a burning sensation. -``` +- This technology PREVIEW is PRE-ALPHA, BLEEDING EDGE, and TOTALLY UNSUPPORTED! +- If this breaks your system, you get to keep both pieces! +- This script is a work in progress and is in constant flux. +- This script is intended to build a quick prototype proof of concept so you can see what our new platform might look like. This configuration will change drastically over time leading up to the final - release. +- Do NOT run this on a system that you care about! +- Do NOT run this on a system that has data that you care about! +- This script should only be run on a TEST box with TEST data! +- This script is only designed for standalone boxes and does NOT support distributed deployments. +- Use of this script may result in nausea, vomiting, or a burning sensation. Installation: From 8775780ffb3b4c8c5ab70f08ed96cad0dd0b4fd3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 15:35:36 -0400 Subject: [PATCH 65/67] Update Readme --- README.md | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index dace6ef32..dca23399a 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ - This script is only designed for standalone boxes and does NOT support distributed deployments. - Use of this script may result in nausea, vomiting, or a burning sensation. -Installation: +**Installation:** If you are using CentOS 7 there are a couple pre-requisites: @@ -28,7 +28,7 @@ git clone https://github.com/Security-Onion-Solutions/securityonion-saltstack cd securityonion-saltstack sudo bash so-setup-network.sh ``` -Allow Access to Kibana: +**Allow Access to Kibana:** For a single host: ``` @@ -41,15 +41,3 @@ sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh analyst 192.168.30.0/24 Then connect to your master via https://YOURMASTER See the [FAQ](https://github.com/Security-Onion-Solutions/securityonion-saltstack/wiki/FAQ) on the Hybrid Hunter wiki. - -**Warnings and Disclaimers** - -This technology PREVIEW is PRE-ALPHA, BLEEDING EDGE, and TOTALLY UNSUPPORTED! -If this breaks your system, you get to keep both pieces! -This script is a work in progress and is in constant flux. -This script is intended to build a quick prototype proof of concept so you can see what our ultimate ELK configuration might look like. This configuration will change drastically over time leading up to the final release. -Do NOT run this on a system that you care about! -Do NOT run this on a system that has data that you care about! -This script should only be run on a TEST box with TEST data! -This script is only designed for standalone boxes and does NOT support distributed deployments. -Use of this script may result in nausea, vomiting, or a burning sensation. From 56502ffde4445bd094f01f47608c412ee6845318 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 15:41:23 -0400 Subject: [PATCH 66/67] Update Readme --- README.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dca23399a..6bae1cece 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,24 @@ - This script is only designed for standalone boxes and does NOT support distributed deployments. - Use of this script may result in nausea, vomiting, or a burning sensation. -**Installation:** +**Requirements** + +Evaluation Mode: + +- Single Ubuntu 16.04 or CentOS 7 VM +- Minimum 8GB of RAM +- Minimum 4 CPU cores +- Minimum 2 NICs + +Distributed: + +- 3 VMs running Ubuntu 16.04 or CentOS 7 (You can mix and match) +- Minimum 8GB of RAM per VM +- Minimum 4 CPU cores per VM +- Minimum 2 NICs for forward nodes + + +**Installation** If you are using CentOS 7 there are a couple pre-requisites: @@ -28,7 +45,7 @@ git clone https://github.com/Security-Onion-Solutions/securityonion-saltstack cd securityonion-saltstack sudo bash so-setup-network.sh ``` -**Allow Access to Kibana:** +**Allow Access to Kibana** For a single host: ``` From f555683643b25f3b43771d0b4b7e96d694f14c84 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 2 Nov 2018 15:46:57 -0400 Subject: [PATCH 67/67] Update Readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 6bae1cece..34c61d33b 100644 --- a/README.md +++ b/README.md @@ -58,3 +58,5 @@ sudo /opt/so/saltstack/pillar/firewall/addfirewall.sh analyst 192.168.30.0/24 Then connect to your master via https://YOURMASTER See the [FAQ](https://github.com/Security-Onion-Solutions/securityonion-saltstack/wiki/FAQ) on the Hybrid Hunter wiki. + +For issues not covered in the FAQ please use the mailing list with the subject prefix of [Hybrid Hunter].