Merge pull request #4800 from Security-Onion-Solutions/hotfix/2.3.60

Update ISO info

HOTFIX

@@ -1 +1 @@
-ECSFIX
+ECSFIX HEAVYNODE_SSL_LOGSTASH_REDIS_PIPELINES FBPIPELINE

VERIFY_ISO.md

@@ -1,18 +1,18 @@
-### 2.3.60-ECSFIX ISO image built on 2021/07/02
+### 2.3.60-FBPIPELINE ISO image built on 2021/07/13
 
 
 
 ### Download and Verify
 
-2.3.60-ECSFIX ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.60-ECSFIX.iso
+2.3.60-FBPIPELINE ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.3.60-FBPIPELINE.iso
 
-MD5: BCD2C449BD3B65D96A0D1E479C0414F9  
-SHA1: 18FB8F33C19980992B291E5A7EC23D5E13853933  
-SHA256: AD3B750E7FC4CA0D58946D8FEB703AE9B01508E314967566B06CFE5D8A8086E9 
+MD5: 2EA2B337289D0CFF0C7488E8E88FE7BE  
+SHA1: 7C22F16AD395E079F4C5345093AF26C105E36D4C  
+SHA256: 3B685BBD19711229C5FCD5D254BA5024AF0C36A3E379790B5E83037CE2668724 
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.60-ECSFIX.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.60-FBPIPELINE.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.60-ECSFIX.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.60-FBPIPELINE.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.60-ECSFIX.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.60-FBPIPELINE.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.60-ECSFIX.iso.sig securityonion-2.3.60-ECSFIX.iso
+gpg --verify securityonion-2.3.60-FBPIPELINE.iso.sig securityonion-2.3.60-FBPIPELINE.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Fri 02 Jul 2021 10:15:04 AM EDT using RSA key ID FE507013
+gpg: Signature made Tue 13 Jul 2021 04:12:08 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

salt/common/tools/sbin/so-airgap-hotfixapply

@@ -1,64 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-. /usr/sbin/so-common
-
-UPDATE_DIR=/tmp/sohotfixapply
-
-if [ -z "$1" ]; then
-  echo "No tarball given. Please provide the filename so I can run the hotfix"
-  echo "so-airgap-hotfixapply /path/to/sohotfix.tar" 
-  exit 1
-else
-  if [ ! -f "$1" ]; then
-    echo "Unable to find $1. Make sure your path is correct and retry."
-    exit 1
-  else
-    echo "Determining if we need to apply this hotfix"
-    rm -rf $UPDATE_DIR
-    mkdir -p $UPDATE_DIR
-    tar xvf $1 -C $UPDATE_DIR
-
-    # Compare some versions
-    NEWVERSION=$(cat $UPDATE_DIR/VERSION)
-    HOTFIXVERSION=$(cat $UPDATE_DIR/HOTFIX)
-    CURRENTHOTFIX=$(cat /etc/sohotfix)
-    INSTALLEDVERSION=$(cat /etc/soversion)
-
-    if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
-      echo "Checking to see if there are hotfixes needed"
-      if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
-        echo "You are already running the latest version of Security Onion."
-        rm -rf $UPDATE_DIR
-        exit 1
-      else
-        echo "We need to apply a hotfix"
-        copy_new_files
-        echo $HOTFIXVERSION > /etc/sohotfix
-        salt-call state.highstate -l info queue=True
-        echo "The Hotfix $HOTFIXVERSION has been applied"
-        # Clean up 
-        rm -rf $UPDATE_DIR
-        exit 0
-      fi
-    else
-      echo "This hotfix is not compatible with your current version. Download the latest ISO and run soup"
-      rm -rf $UPDATE_DIR
-    fi
-
-  fi
-fi

salt/common/tools/sbin/so-airgap-hotfixdownload

@@ -1,33 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# Get the latest code
-rm -rf /tmp/sohotfix
-mkdir -p /tmp/sohotfix
-cd /tmp/sohotfix
-git clone https://github.com/Security-Onion-Solutions/securityonion
-if [ ! -d "/tmp/sohotfix/securityonion" ]; then
-  echo "I was unable to get the latest code. Check your internet and try again."
-  exit 1
-else
-  echo "Looks like we have the code lets create the tarball."
-  cd /tmp/sohotfix/securityonion
-  tar cvf /tmp/sohotfix/sohotfix.tar HOTFIX VERSION salt pillar
-  echo ""
-  echo "Copy /tmp/sohotfix/sohotfix.tar to portable media and then copy it to your airgap manager."
-  exit 0
-fi

salt/logstash/init.sls

@@ -36,6 +36,14 @@
 {% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %}
 {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
 
+{% if grains.role in ['so-heavynode'] %}
+  {% set EXTRAHOSTHOSTNAME = salt['grains.get']('host') %}
+  {% set EXTRAHOSTIP = salt['pillar.get']('sensor:mainip') %}
+{% else %}
+  {% set EXTRAHOSTHOSTNAME = MANAGER %}
+  {% set EXTRAHOSTIP = MANAGERIP %}
+{% endif %}
+
 include:
   - elasticsearch
 
@@ -145,7 +153,7 @@ so-logstash:
     - name: so-logstash
     - user: logstash
     - extra_hosts:
-      - {{ MANAGER }}:{{ MANAGERIP }}
+      - {{ EXTRAHOSTHOSTNAME }}:{{ EXTRAHOSTIP }}
     - environment:
       - LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }}
     - port_bindings:

salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja

@@ -1,10 +1,13 @@
-{%- set MANAGER = salt['grains.get']('master') %}
+{%- if grains.role in ['so-heavynode'] %}
+  {%- set HOST = salt['grains.get']('host') %}
+{%- else %}
+  {%- set HOST = salt['grains.get']('master') %}
+{%- endif %}
   {%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
-{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
-
+{%- set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
 input {
 	redis {
-		host => '{{ MANAGER }}'
+		host => '{{ HOST }}'
 		port => 9696
 		ssl => true
 		data_type => 'list'

salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja

@@ -6,7 +6,7 @@
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
-  if [metadata][pipeline] {
+  if "filebeat" in [metadata][pipeline] {
     elasticsearch {
       id => "filebeat_modules_metadata_pipeline"
       pipeline => "%{[metadata][pipeline]}"

salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja

@@ -1,8 +1,12 @@
-{%- set MANAGER = salt['grains.get']('master') %}
-{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
+{%- if grains.role in ['so-heavynode'] %}
+  {%- set HOST = salt['grains.get']('host') %}
+{%- else %}
+  {%- set HOST = salt['grains.get']('master') %}
+{%- endif %}
+{%- set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
 output {
 	redis {
-		host => '{{ MANAGER }}'
+		host => '{{ HOST }}'
 		port => 6379
 		data_type => 'list'
 		key => 'logstash:unparsed'

salt/ssl/init.sls

@@ -9,6 +9,11 @@
 {% set MAININT = salt['pillar.get']('host:mainint') %}
 {% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %}
 {% set CUSTOM_FLEET_HOSTNAME = salt['pillar.get']('global:fleet_custom_hostname', None) %}
+{% if grains.role in ['so-heavynode'] %}
+  {% set COMMONNAME = salt['grains.get']('host') %}
+{% else %}
+  {% set COMMONNAME = manager %}
+{% endif %}
 
 {% if grains.id.split('_')|last in ['manager', 'eval', 'standalone', 'import', 'helixsensor'] %}
     {% set trusttheca_text = salt['cp.get_file_str']('/etc/pki/ca.crt')|replace('\n', '') %}
@@ -83,10 +88,12 @@ removeesp12dir:
     - days_remaining: 0
     - days_valid: 820
     - backup: True
+{% if grains.role not in ['so-heavynode'] %}
     - unless:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/influxdb.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+{% endif %}
     - timeout: 30
     - retry:
         attempts: 5
@@ -103,7 +110,7 @@ influxkeyperms:
 # Create a cert for Redis encryption
 /etc/pki/redis.key:
   x509.private_key_managed:
-    - CN: {{ manager }}
+    - CN: {{ COMMONNAME }}
     - bits: 4096
     - days_remaining: 0
     - days_valid: 820
@@ -123,14 +130,16 @@ influxkeyperms:
     - ca_server: {{ ca_server }}
     - signing_policy: registry
     - public_key: /etc/pki/redis.key
-    - CN: {{ manager }}
+    - CN: {{ COMMONNAME }}
     - days_remaining: 0
     - days_valid: 820
     - backup: True
+{% if grains.role not in ['so-heavynode'] %}
     - unless:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/redis.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+{% endif %}
     - timeout: 30
     - retry:
         attempts: 5
@@ -147,7 +156,7 @@ rediskeyperms:
 {% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode'] %}
 /etc/pki/filebeat.key:
   x509.private_key_managed:
-    - CN: {{ manager }}
+    - CN: {{ COMMONNAME }}
     - bits: 4096
     - days_remaining: 0
     - days_valid: 820
@@ -168,18 +177,16 @@ rediskeyperms:
     - ca_server: {{ ca_server }}
     - signing_policy: filebeat
     - public_key: /etc/pki/filebeat.key
-{% if grains.role == 'so-heavynode' %}
-    - CN: {{grains.host}}
-{% else %}
-    - CN: {{manager}}
-{% endif %}
+    - CN: {{ COMMONNAME }}
     - days_remaining: 0
     - days_valid: 820
     - backup: True
+{% if grains.role not in ['so-heavynode'] %}
     - unless:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/filebeat.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+{% endif %}
     - timeout: 30
     - retry:
         attempts: 5
@@ -315,7 +322,7 @@ miniokeyperms:
 # Create a cert for elasticsearch
 /etc/pki/elasticsearch.key:
   x509.private_key_managed:
-    - CN: {{ manager }}
+    - CN: {{ COMMONNAME }}
     - bits: 4096
     - days_remaining: 0
     - days_valid: 820
@@ -335,14 +342,16 @@ miniokeyperms:
     - ca_server: {{ ca_server }}
     - signing_policy: registry
     - public_key: /etc/pki/elasticsearch.key
-    - CN: {{ manager }}
+    - CN: {{ COMMONNAME }}
     - days_remaining: 0
     - days_valid: 820
     - backup: True
+{% if grains.role not in ['so-heavynode'] %}
     - unless:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+{% endif %}
     - timeout: 30
     - retry:
         attempts: 5
@@ -462,7 +471,7 @@ fbcertdir:
 
 /opt/so/conf/filebeat/etc/pki/filebeat.key:
   x509.private_key_managed:
-    - CN: {{ manager }}
+    - CN: {{ COMMONNAME }}
     - bits: 4096
     - days_remaining: 0
     - days_valid: 820
@@ -483,18 +492,16 @@ fbcertdir:
     - ca_server: {{ ca_server }}
     - signing_policy: filebeat
     - public_key: /opt/so/conf/filebeat/etc/pki/filebeat.key
-{% if grains.role == 'so-heavynode' %}
-    - CN: {{grains.id}}
-{% else %}
-    - CN: {{manager}}
-{% endif %}
+    - CN: {{ COMMONNAME }}
     - days_remaining: 0
     - days_valid: 820
     - backup: True
+{% if grains.role not in ['so-heavynode'] %}
     - unless:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /opt/so/conf/filebeat/etc/pki/filebeat.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+{% endif %}
     - timeout: 30
     - retry:
         attempts: 5