diff --git a/VERSION b/VERSION index 197c4d5c2..005119baa 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0 +2.4.1 diff --git a/salt/ca/init.sls b/salt/ca/init.sls index c857b331e..4c7973cd0 100644 --- a/salt/ca/init.sls +++ b/salt/ca/init.sls @@ -18,7 +18,7 @@ include: pki_private_key: x509.private_key_managed: - name: /etc/pki/ca.key - - bits: 4096 + - keysize: 4096 - passphrase: - cipher: aes_256_cbc - backup: True @@ -39,7 +39,7 @@ pki_public_ca_crt: - keyUsage: "critical cRLSign, keyCertSign" - extendedkeyUsage: "serverAuth, clientAuth" - subjectKeyIdentifier: hash - - authorityKeyIdentifier: keyid,issuer:always + - authorityKeyIdentifier: keyid:always, issuer - days_valid: 3650 - days_remaining: 0 - backup: True diff --git a/salt/common/packages.sls b/salt/common/packages.sls index 2ed82c895..9e118f4da 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -52,5 +52,6 @@ commonpkgs: - rsync - python3-rich - python3-watchdog + - python3-packaging - unzip {% endif %} diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 49b0d0527..ea5cc703e 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -16,6 +16,11 @@ if [ -z $NOROOT ]; then fi fi +# Ensure /usr/sbin is in path +if ! echo "$PATH" | grep -q "/usr/sbin"; then + export PATH="$PATH:/usr/sbin" +fi + # Define a banner to separate sections banner="=========================================================================" @@ -49,33 +54,37 @@ add_interface_bond0() { ethtool -K "$BNIC" $i off &>/dev/null fi done - # Check if the bond slave connection has already been created - nmcli -f name,uuid -p con | grep -q "bond0-slave-$BNIC" - local found_int=$? - if [[ $found_int != 0 ]]; then - # Create the slave interface and assign it to the bond - nmcli con add type ethernet ifname "$BNIC" con-name "bond0-slave-$BNIC" master bond0 -- \ - ethernet.mtu "$MTU" \ - connection.autoconnect "yes" - else - local int_uuid - int_uuid=$(nmcli -f name,uuid -p con | sed -n "s/bond0-slave-$BNIC //p" | tr -d ' ') + if ! [[ $is_cloud ]]; then + # Check if the bond slave connection has already been created + nmcli -f name,uuid -p con | grep -q "bond0-slave-$BNIC" + local found_int=$? - nmcli con mod "$int_uuid" \ - ethernet.mtu "$MTU" \ - connection.autoconnect "yes" - fi + if [[ $found_int != 0 ]]; then + # Create the slave interface and assign it to the bond + nmcli con add type ethernet ifname "$BNIC" con-name "bond0-slave-$BNIC" master bond0 -- \ + ethernet.mtu "$MTU" \ + connection.autoconnect "yes" + else + local int_uuid + int_uuid=$(nmcli -f name,uuid -p con | sed -n "s/bond0-slave-$BNIC //p" | tr -d ' ') + + nmcli con mod "$int_uuid" \ + ethernet.mtu "$MTU" \ + connection.autoconnect "yes" + fi + fi ip link set dev "$BNIC" arp off multicast off allmulticast off promisc on - - # Bring the slave interface up - if [[ $verbose == true ]]; then - nmcli con up "bond0-slave-$BNIC" - else - nmcli con up "bond0-slave-$BNIC" &>/dev/null + + if ! [[ $is_cloud ]]; then + # Bring the slave interface up + if [[ $verbose == true ]]; then + nmcli con up "bond0-slave-$BNIC" + else + nmcli con up "bond0-slave-$BNIC" &>/dev/null + fi fi - if [ "$nic_error" != 0 ]; then return "$nic_error" fi @@ -196,7 +205,7 @@ gpg_rpm_import() { local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/rocky/keys" fi - RPMKEYS=('RPM-GPG-KEY-EPEL-9' 'SALTSTACK-GPG-KEY2.pub' 'docker.pub' 'securityonion.pub') + RPMKEYS=('RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub') for RPMKEY in "${RPMKEYS[@]}"; do rpm --import $RPMKEYSLOC/$RPMKEY diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete index fe21daae0..70b483424 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete +++ b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list index 479fdc72d..d81067a7e 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list +++ b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view index 5f267220b..5e5b62de0 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view +++ b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the diff --git a/salt/common/tools/sbin/so-elastic-fleet-data-streams-list b/salt/common/tools/sbin/so-elastic-fleet-data-streams-list index 238fc8ada..b3e35fdba 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-data-streams-list +++ b/salt/common/tools/sbin/so-elastic-fleet-data-streams-list @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete index 4596774b4..ac600ab40 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete index a8601028c..ded8da808 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list index 07d2822fb..9dffc613c 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index be4ec5246..4efdd5784 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the @@ -134,4 +134,4 @@ echo echo echo "Setting up SOC - Salt_Relay Logs package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"package":{"name":"log","version":"1.1.2"},"name":"soc-salt-relay-logs","namespace":"so","description":"Security Onion - Salt Relay - Logs","policy_id":"so-grid-nodes","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/opt/so/log/soc/salt-relay.log"],"data_stream.dataset":"soc","custom":"pipeline: common","processors": "- dissect:\n tokenizer: \"%{soc.ts} | %{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: salt_relay","tags":[]}}}}}}' -echo \ No newline at end of file +echo diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index 5eeccb8db..ab54d42a5 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -97,9 +97,9 @@ salt-call state.apply elasticfleet queue=True /usr/sbin/so-elastic-fleet-integration-policy-load # Temp -wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.6.2/so-elastic-agent-8.6.2-darwin-x86_64.tar.gz -wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.6.2/so-elastic-agent-8.6.2-linux-x86_64.tar.gz -wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.6.2/so-elastic-agent-8.6.2-windows-x86_64.tar.gz +wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.7.0/so-elastic-agent-8.7.0-darwin-x86_64.tar.gz +wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.7.0/so-elastic-agent-8.7.0-linux-x86_64.tar.gz +wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.7.0/so-elastic-agent-8.7.0-windows-x86_64.tar.gz #git clone -b 2.4-so-elastic-agent https://github.com/Security-Onion-Solutions/securityonion-image.git #cd securityonion-image/so-elastic-agent-builder diff --git a/salt/common/tools/sbin/so-elasticsearch-cluster-space-total b/salt/common/tools/sbin/so-elasticsearch-cluster-space-total new file mode 100755 index 000000000..3faa2a7a9 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-cluster-space-total @@ -0,0 +1,57 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +{% from 'vars/globals.map.jinja' import GLOBALS %} + +TOTAL_AVAILABLE_SPACE=0 + +# Wait for ElasticSearch to initialize +COUNT=0 +ELASTICSEARCH_CONNECTED="no" +while [[ "$COUNT" -le 240 ]]; do + /usr/sbin/so-elasticsearch-query / -k --output /dev/null --silent --head --fail + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + break + else + ((COUNT+=1)) + sleep 1 + fi +done +if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo + exit 1 +fi + +# Set percentage of space to desired value, otherwise use a default value of 80 percent +if [[ "$1" != "" ]]; then + PERCENTAGE=$1 +else + PERCENTAGE=80 +fi + +# Iterate through the output of _cat/allocation for each node in the cluster to determine the total available space +{% if GLOBALS.role == 'so-manager' %} +for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v {{ GLOBALS.manager }} | awk '{print $5}'); do +{% else %} +for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $5}'); do +{% endif %} + size=$(echo $i | grep -oE '[0-9].*' | awk '{print int($1+0.5)}') + unit=$(echo $i | grep -oE '[A-Za-z]+') + if [ $unit = "tb" ]; then + size=$(( size * 1024 )) + fi + TOTAL_AVAILABLE_SPACE=$(( TOTAL_AVAILABLE_SPACE + size )) +done + +# Calculate the percentage of available space based on our previously defined value +PERCENTAGE_AVAILABLE_SPACE=$(( TOTAL_AVAILABLE_SPACE*PERCENTAGE/100 )) +echo "$PERCENTAGE_AVAILABLE_SPACE" diff --git a/salt/common/tools/sbin/so-elasticsearch-cluster-space-used b/salt/common/tools/sbin/so-elasticsearch-cluster-space-used new file mode 100755 index 000000000..b8ac4f6e6 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-cluster-space-used @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common +{% from 'vars/globals.map.jinja' import GLOBALS %} + +TOTAL_AVAILABLE_SPACE=0 + +# Iterate through the output of _cat/allocation for each node in the cluster to determine the total available space +{% if GLOBALS.role == 'so-manager' %} +for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v {{ GLOBALS.manager }} | awk '{print $3}'); do +{% else %} +for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $3}'); do +{% endif %} + size=$(echo $i | grep -oE '[0-9].*' | awk '{print int($1+0.5)}') + unit=$(echo $i | grep -oE '[A-Za-z]+') + if [ $unit = "tb" ]; then + size=$(( size * 1024 )) + fi + TOTAL_AVAILABLE_SPACE=$(( TOTAL_AVAILABLE_SPACE + size )) +done + +# Calculate the percentage of available space based on our previously defined value +echo "$TOTAL_AVAILABLE_SPACE" diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-lifecycle-status b/salt/common/tools/sbin/so-elasticsearch-ilm-lifecycle-status index db31dcb0f..130a7cf16 100755 --- a/salt/common/tools/sbin/so-elasticsearch-ilm-lifecycle-status +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-lifecycle-status @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-delete b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-delete index 108dd1178..2be9dabb2 100755 --- a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-delete +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-delete @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load index dda521736..26ce487a7 100755 --- a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-view b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-view index d69e328fe..426b6938d 100755 --- a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-view +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-policy-view @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-restart b/salt/common/tools/sbin/so-elasticsearch-ilm-restart index 7f422ed6e..54654b251 100755 --- a/salt/common/tools/sbin/so-elasticsearch-ilm-restart +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-restart @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-stop b/salt/common/tools/sbin/so-elasticsearch-ilm-stop index 23c068918..4868fd86d 100755 --- a/salt/common/tools/sbin/so-elasticsearch-ilm-stop +++ b/salt/common/tools/sbin/so-elasticsearch-ilm-stop @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index aaa43cd9e..3c712491a 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -13,74 +13,74 @@ DEFAULT_ROLE=analyst function usage() { cat < [supporting parameters]" + Usage: $0 [supporting parameters] - where is one of the following:" + where is one of the following: - list: Lists all user email addresses currently defined in the identity system" + list: Lists all user email addresses currently defined in the identity system - add: Adds a new user to the identity system" - Required parameters: " - --email " - Optional parameters: " - --role (defaults to $DEFAULT_ROLE)" - --firstName (defaults to blank)" - --lastName (defaults to blank)" - --note (defaults to blank)" + add: Adds a new user to the identity system + Required parameters: + --email + Optional parameters: + --role (defaults to $DEFAULT_ROLE) + --firstName (defaults to blank) + --lastName (defaults to blank) + --note (defaults to blank) --skip-sync (defers the Elastic sync until the next scheduled time) - addrole: Grants a role to an existing user" - Required parameters: " - --email " - --role " - Optional parameters: " + addrole: Grants a role to an existing user + Required parameters: + --email + --role + Optional parameters: --skip-sync (defers the Elastic sync until the next scheduled time) - delrole: Removes a role from an existing user" - Required parameters: " - --email " - --role " - Optional parameters: " + delrole: Removes a role from an existing user + Required parameters: + --email + --role + Optional parameters: --skip-sync (defers the Elastic sync until the next scheduled time) - password: Updates a user's password and disables MFA" - Required parameters: " - --email " - Optional parameters: " + password: Updates a user's password and disables MFA + Required parameters: + --email + Optional parameters: --skip-sync (defers the Elastic sync until the next scheduled time) - profile: Updates a user's profile information" - Required parameters: " - --email " - Optional parameters: " - --role (defaults to $DEFAULT_ROLE)" - --firstName (defaults to blank)" - --lastName (defaults to blank)" - --note (defaults to blank)" + profile: Updates a user's profile information + Required parameters: + --email + Optional parameters: + --role (defaults to $DEFAULT_ROLE) + --firstName (defaults to blank) + --lastName (defaults to blank) + --note (defaults to blank) - enable: Enables a user" - Required parameters: " - --email " - Optional parameters: " + enable: Enables a user + Required parameters: + --email + Optional parameters: --skip-sync (defers the Elastic sync until the next scheduled time) - disable: Disables a user" - Required parameters: " - --email " - Optional parameters: " + disable: Disables a user + Required parameters: + --email + Optional parameters: --skip-sync (defers the Elastic sync until the next scheduled time) - validate: Validates that the given email address and password are acceptable" - Required parameters: " - --email " + validate: Validates that the given email address and password are acceptable + Required parameters: + --email - valemail: Validates that the given email address is acceptable; requires 'email' parameter" - Required parameters: " - --email " + valemail: Validates that the given email address is acceptable; requires 'email' parameter + Required parameters: + --email - valpass: Validates that a password is acceptable" + valpass: Validates that a password is acceptable - Note that the password can be piped into STDIN to avoid prompting for it" + Note that the password can be piped into STDIN to avoid prompting for it USAGE_EOF exit 1 } diff --git a/salt/curator/files/action/delete.yml b/salt/curator/files/action/delete.yml index fb8ae30cb..c81a9e548 100644 --- a/salt/curator/files/action/delete.yml +++ b/salt/curator/files/action/delete.yml @@ -3,6 +3,11 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICDEFAULTS %} +{% set ELASTICMERGED = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) %} + +{{ ELASTICMERGED.retention_pct }} + {%- set log_size_limit = salt['pillar.get']('elasticsearch:log_size_limit') %} actions: 1: diff --git a/salt/curator/files/bin/so-curator-closed-delete b/salt/curator/files/bin/so-curator-closed-delete deleted file mode 100755 index e585df406..000000000 --- a/salt/curator/files/bin/so-curator-closed-delete +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -#. /usr/sbin/so-elastic-common -#. /etc/nsm/securityonion.conf - -# If logrotate script doesn't already exist, create it -#FILE="/etc/logrotate.d/so-curator-closed-delete" -#if ! [ -f ${FILE} ]; then -# cat << EOF > ${FILE} -#/var/log/nsm/so-curator-closed-delete.log { -# daily -# rotate 7 -# copytruncate -# compress -# missingok -# notifempty -#} -#EOF -#fi - -# Avoid starting multiple instances -APP=closeddelete -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf - -/usr/sbin/so-curator-closed-delete-delete diff --git a/salt/curator/files/bin/so-curator-closed-delete-delete b/salt/curator/files/bin/so-curator-closed-delete-delete deleted file mode 100755 index fe62d3bf5..000000000 --- a/salt/curator/files/bin/so-curator-closed-delete-delete +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'vars/globals.map.jinja' import GLOBALS %} -{%- if grains['role'] in ['so-searchnode', 'so-heavynode'] %} - {%- set ELASTICSEARCH_HOST = GLOBALS.node_ip -%} - {%- set ELASTICSEARCH_PORT = salt['pillar.get']('elasticsearch:es_port') -%} -{%- elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %} - {%- set ELASTICSEARCH_HOST = GLOBALS.manager_ip -%} - {%- set ELASTICSEARCH_PORT = salt['pillar.get']('manager:es_port') -%} -{%- endif -%} -{%- set LOG_SIZE_LIMIT = salt['pillar.get']('elasticsearch:log_size_limit') -%} - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -LOG="/opt/so/log/curator/so-curator-closed-delete.log" - -overlimit() { - - [[ $(du -hs --block-size=1GB /nsm/elasticsearch/nodes | awk '{print $1}' ) -gt "{{LOG_SIZE_LIMIT}}" ]] -} - -closedindices() { - - # If we can't query Elasticsearch, then immediately return false. - curl -K /opt/so/conf/elasticsearch/curl.config -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed >/dev/null 2>&1 - [ $? -eq 1 ] && return false - # First, get the list of closed indices using _cat/indices?h=index\&expand_wildcards=closed. - # Next, filter out any so-case indices. - # Finally, use grep's -q option to return true if there are any remaining logstash- or so- indices. - curl -K /opt/so/conf/elasticsearch/curl.config -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -v "so-case" | grep -q -E "(logstash-|so-)" -} - -# Check for 2 conditions: -# 1. Are Elasticsearch indices using more disk space than LOG_SIZE_LIMIT? -# 2. Are there any closed indices that we can delete? -# If both conditions are true, keep on looping until one of the conditions is false. -while overlimit && closedindices; do - - # We need to determine OLDEST_INDEX: - # First, get the list of closed indices using _cat/indices?h=index\&expand_wildcards=closed. - # Next, filter out any so-case indices and only select the remaining logstash- or so- indices. - # Then, sort by date by telling sort to use hyphen as delimiter and sort on the third field. - # Finally, select the first entry in that sorted list. - OLDEST_INDEX=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -v "so-case" | grep -E "(logstash-|so-)" | sort -t- -k3 | head -1) - - # Now that we've determined OLDEST_INDEX, ask Elasticsearch to delete it. - curl -K /opt/so/conf/elasticsearch/curl.config-XDELETE -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX} - - # Finally, write a log entry that says we deleted it. - echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT ({{LOG_SIZE_LIMIT}} GB) - Index ${OLDEST_INDEX} deleted ..." >> ${LOG} - -done diff --git a/salt/curator/files/bin/so-curator-cluster-close b/salt/curator/files/bin/so-curator-cluster-close old mode 100644 new mode 100755 diff --git a/salt/curator/files/bin/so-curator-cluster-delete b/salt/curator/files/bin/so-curator-cluster-delete old mode 100644 new mode 100755 index 34c3c10cf..0f7945b78 --- a/salt/curator/files/bin/so-curator-cluster-delete +++ b/salt/curator/files/bin/so-curator-cluster-delete @@ -4,7 +4,8 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -APP=delete +# Avoid starting multiple instances +APP=clusterdelete lf=/tmp/$APP-pidLockFile # create empty lock file if none exists cat /dev/null >> $lf @@ -13,18 +14,4 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-delete.yml > /dev/null 2>&1; -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-delete.yml > /dev/null 2>&1; -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-delete.yml > /dev/null 2>&1; -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-delete.yml > /dev/null 2>&1; -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-delete.yml > /dev/null 2>&1; -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-kratos-delete.yml > /dev/null 2>&1; -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-delete.yml > /dev/null 2>&1; -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-delete.yml > /dev/null 2>&1; -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-delete.yml > /dev/null 2>&1; -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-delete.yml > /dev/null 2>&1; -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-import-so-delete.yml > /dev/null 2>&1; -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-strelka-delete.yml > /dev/null 2>&1; -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-suricata-delete.yml > /dev/null 2>&1; -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-syslog-delete.yml > /dev/null 2>&1; -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-zeek-delete.yml > /dev/null 2>&1; +/usr/sbin/so-curator-cluster-delete-delete diff --git a/salt/curator/files/bin/so-curator-cluster-delete-delete b/salt/curator/files/bin/so-curator-cluster-delete-delete new file mode 100755 index 000000000..fe41ca1f0 --- /dev/null +++ b/salt/curator/files/bin/so-curator-cluster-delete-delete @@ -0,0 +1,85 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICDEFAULTS %} +{%- set ELASTICSEARCH_HOST = GLOBALS.node_ip -%} +{%- set RETENTION = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) -%} + +LOG="/opt/so/log/curator/so-curator-cluster-delete.log" +LOG_SIZE_LIMIT=$(/usr/sbin/so-elasticsearch-cluster-space-total {{ RETENTION.retention_pct}}) + +overlimit() { + [[ $(/usr/sbin/so-elasticsearch-cluster-space-used) -gt "${LOG_SIZE_LIMIT}" ]] +} + +closedindices() { + # If we can't query Elasticsearch, then immediately return false. + /usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep close > /dev/null 2>&1 + [ $? -eq 1 ] && return false + # First, get the list of closed indices using _cat/indices?h=index,status | grep close | awk '{print $1}'. + # Next, filter out any so-case indices. + # Finally, use grep's -q option to return true if there are any remaining logstash-, so-, or .ds-logs- indices. + /usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep close | awk '{print $1}' | grep -v "so-case" | grep -q -E "(logstash-|so-|.ds-logs-)" +} + +# Check for 2 conditions: +# 1. Are Elasticsearch indices using more disk space than LOG_SIZE_LIMIT? +# 2. Are there any closed indices that we can delete? +# If both conditions are true, keep on looping until one of the conditions is false. + +while overlimit && closedindices; do + CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep close | awk '{print $1}' | grep -v "so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3) + # We iterate through the closed indices + for CLOSED_INDEX in ${CLOSED_INDICES}; do + # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream + # To do so, we need to identify to which data stream this index is associated + # We extract the data stream name using the pattern below + DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+" + DATASTREAM=$(echo "${CLOSED_INDEX}" | grep -oE "$DATASTREAM_PATTERN") + # We look up the data stream, and determine the write index + CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name) + # We make sure we are not trying to delete a write index + if [ "${CLOSED_INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then + # This should not be a write index, so we should be allowed to delete it + /usr/sbin/so-elasticsearch-query ${CLOSED_INDEX} -XDELETE + # Finally, write a log entry that says we deleted it. + echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT} GB) - Index ${CLOSED_INDEX} deleted ..." >> ${LOG} + fi + if ! overlimit; then + exit + fi + done +done + +while overlimit; do + + # We need to determine the oldest open index. + # First, get the list of open indices using _cat/indices?h=index,status | grep open | awk '{print $1}'. + # Next, filter out any so-case indices and only select the remaining logstash-, so-, or .ds-logs- indices. + # Then, sort by date by telling sort to use hyphen as delimiter and sort on the third field. + OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep open | awk '{print $1}' | grep -v "so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3) + # We iterate through the open indices + for OPEN_INDEX in ${OPEN_INDICES}; do + # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream + # To do so, we need to identify to which data stream this index is associated + # We extract the data stream name using the pattern below + DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+" + DATASTREAM=$(echo "${OPEN_INDEX}" | grep -oE "$DATASTREAM_PATTERN") + # We look up the data stream, and determine the write index + CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name) + # We make sure we are not trying to delete a write index + if [ "${OPEN_INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then + # This should not be a write index, so we should be allowed to delete it + /usr/sbin/so-elasticsearch-query ${OPEN_INDEX} -XDELETE + # Finally, write a log entry that says we deleted it. + echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT} GB) - Index ${OPEN_INDEX} deleted ..." >> ${LOG} + fi + if ! overlimit; then + exit + fi + done +done diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 94a666f53..36f1261fe 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -27,6 +27,12 @@ curator: - createhome: False # Create the log directory +curlogdir: + file.directory: + - name: /opt/so/log/curator + - user: 934 + - group: 939 + curactiondir: file.directory: - name: /opt/so/conf/curator/action @@ -34,12 +40,6 @@ curactiondir: - group: 939 - makedirs: True -curlogdir: - file.directory: - - name: /opt/so/log/curator - - user: 934 - - group: 939 - actionconfs: file.recurse: - name: /opt/so/conf/curator/action @@ -50,7 +50,6 @@ actionconfs: - defaults: CURATORMERGED: {{ CURATORMERGED }} - curconf: file.managed: - name: /opt/so/conf/curator/curator.yml @@ -61,40 +60,6 @@ curconf: - template: jinja - show_changes: False -curcloseddel: - file.managed: - - name: /usr/sbin/so-curator-closed-delete - - source: salt://curator/files/bin/so-curator-closed-delete - - user: 934 - - group: 939 - - mode: 755 - -curcloseddeldel: - file.managed: - - name: /usr/sbin/so-curator-closed-delete-delete - - source: salt://curator/files/bin/so-curator-closed-delete-delete - - user: 934 - - group: 939 - - mode: 755 - - template: jinja - -curclose: - file.managed: - - name: /usr/sbin/so-curator-close - - source: salt://curator/files/bin/so-curator-close - - user: 934 - - group: 939 - - mode: 755 - - template: jinja - -curdel: - file.managed: - - name: /usr/sbin/so-curator-delete - - source: salt://curator/files/bin/so-curator-delete - - user: 934 - - group: 939 - - mode: 755 - curclusterclose: file.managed: - name: /usr/sbin/so-curator-cluster-close @@ -104,13 +69,21 @@ curclusterclose: - mode: 755 - template: jinja -curclusterdelete: +curclusterdelete: file.managed: - name: /usr/sbin/so-curator-cluster-delete - source: salt://curator/files/bin/so-curator-cluster-delete - user: 934 - group: 939 - mode: 755 + +curclusterdeletedelete: + file.managed: + - name: /usr/sbin/so-curator-cluster-delete-delete + - source: salt://curator/files/bin/so-curator-cluster-delete-delete + - user: 934 + - group: 939 + - mode: 755 - template: jinja so-curator: @@ -163,12 +136,12 @@ so-curatorclusterclose: - month: '*' - dayweek: '*' -so-curatorclusterdelete: +so-curatorclusterdeletecron: cron.present: - - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-delete.log 2>&1 + - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-cluster-delete.log 2>&1 - user: root - - minute: '2' - - hour: '*/1' + - minute: '*/5' + - hour: '*' - daymonth: '*' - month: '*' - dayweek: '*' diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 3649d015e..37f55ddc7 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,4 +1,6 @@ elasticsearch: + retention: + retention_pct: 50 config: node: {} cluster: diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index b9caa2e89..e128c9dae 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -407,7 +407,6 @@ so-elasticsearch-roles-load: - docker_container: so-elasticsearch - file: es_sync_scripts - {% else %} {{sls}}_state_not_allowed: diff --git a/salt/elasticsearch/roles/analyst.json b/salt/elasticsearch/roles/analyst.json index fc788e2f1..2fd10ab47 100644 --- a/salt/elasticsearch/roles/analyst.json +++ b/salt/elasticsearch/roles/analyst.json @@ -18,6 +18,7 @@ "indices": [ { "names": [ + "logs-*", "so-*" ], "privileges": [ @@ -60,4 +61,4 @@ } ], "run_as": [] -} \ No newline at end of file +} diff --git a/salt/elasticsearch/roles/auditor.json b/salt/elasticsearch/roles/auditor.json index 26df1207c..6669bd2f2 100644 --- a/salt/elasticsearch/roles/auditor.json +++ b/salt/elasticsearch/roles/auditor.json @@ -16,6 +16,7 @@ "indices": [ { "names": [ + "logs-*", "so-*" ], "privileges": [ @@ -56,4 +57,4 @@ } ], "run_as": [] -} \ No newline at end of file +} diff --git a/salt/elasticsearch/roles/limited-analyst.json b/salt/elasticsearch/roles/limited-analyst.json index 2b3797dbc..9186b732e 100644 --- a/salt/elasticsearch/roles/limited-analyst.json +++ b/salt/elasticsearch/roles/limited-analyst.json @@ -4,6 +4,7 @@ "indices": [ { "names": [ + "logs-*", "so-*" ], "privileges": [ @@ -46,4 +47,4 @@ } ], "run_as": [] -} \ No newline at end of file +} diff --git a/salt/elasticsearch/roles/limited-auditor.json b/salt/elasticsearch/roles/limited-auditor.json index ecab5016a..902567523 100644 --- a/salt/elasticsearch/roles/limited-auditor.json +++ b/salt/elasticsearch/roles/limited-auditor.json @@ -4,6 +4,7 @@ "indices": [ { "names": [ + "logs-*", "so-*" ], "privileges": [ @@ -44,4 +45,4 @@ } ], "run_as": [] -} \ No newline at end of file +} diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index d7c310687..ded8e5be6 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -1,7 +1,12 @@ elasticsearch: esheap: description: Specify the memory heap size in (m)egabytes for Elasticsearch. - helpLink: elasticsearch.html + helpLink: elasticsearch.html + retention: + retention_pct: + decription: Total percentage of space used by Elasticsearch for multi node clusters + helpLink: elasticsearch.yaml + global: True config: cluster: name: diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-settings b/salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-settings index 0cdec181e..83a997845 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-settings +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-settings @@ -1,4 +1,10 @@ #!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common ELASTICSEARCH_PORT=9200 diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load index dda521736..26ce487a7 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines index 04cd86c23..7fdc6ff7b 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines @@ -1,10 +1,10 @@ #!/bin/bash -# # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +. /usr/sbin/so-common RETURN_CODE=0 @@ -51,4 +51,4 @@ if [ ! -f /opt/so/state/espipelines.txt ]; then fi else exit $RETURN_CODE -fi \ No newline at end of file +fi diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load index 130e0f762..17265a7c4 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load @@ -1,9 +1,11 @@ #!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +. /usr/sbin/so-common + default_conf_dir=/opt/so/conf ELASTICSEARCH_HOST="{{ GLOBALS.node_ip }}" ELASTICSEARCH_PORT=9200 diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load index a10d07e13..386026f0c 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -1,7 +1,10 @@ #!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use -# this file except in compliance with the Elastic License 2.0. +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common default_conf_dir=/opt/so/conf ELASTICSEARCH_HOST="{{ GLOBALS.node_ip }}" diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index 63ad170f2..12466cae1 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.6.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.7.0" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index cc25b61e4..e2eced11e 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.6.2","id": "8.6.2","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.7.0","id": "8.7.0","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} diff --git a/salt/repo/client/files/rocky/keys/SALT-PROJECT-GPG-PUBKEY-2023.pub b/salt/repo/client/files/rocky/keys/SALT-PROJECT-GPG-PUBKEY-2023.pub new file mode 100644 index 000000000..be55ef561 --- /dev/null +++ b/salt/repo/client/files/rocky/keys/SALT-PROJECT-GPG-PUBKEY-2023.pub @@ -0,0 +1,41 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBGPazmABDAC6qc2st6/Uh/5AL325OB5+Z1XMFM2HhQNjB/VcYbLvcCx9AXsU +eaEmNPm6OY3p5+j8omjpXPYSU7DUQ0lIutuAtwkDMROH7uH/r9IY7iu88S6w3q89 +bgbnqhu4mrSik2RNH2NqEiJkylz5rwj4F387y+UGH3aXIGryr+Lux9WxfqoRRX7J +WCf6KOaduLSp9lF4qdpAb4/Z5yExXtQRA9HULSJZqNVhfhWInTkVPw+vUo/P9AYv +mJVv6HRNlTb4HCnl6AZGcAYv66J7iWukavmYKxuIbdn4gBJwE0shU9SaP70dh/LT +WqIUuGRZBVH/LCuVGzglGYDh2iiOvR7YRMKf26/9xlR0SpeU/B1g6tRu3p+7OgjA +vJFws+bGSPed07asam3mRZ0Y9QLCXMouWhQZQpx7Or1pUl5Wljhe2W84MfW+Ph6T +yUm/j0yRlZJ750rGfDKA5gKIlTUXr+nTvsK3nnRiHGH2zwrC1BkPG8K6MLRluU/J +ChgZo72AOpVNq9MAEQEAAbQ5U2FsdCBQcm9qZWN0IFBhY2thZ2luZyA8c2FsdHBy +b2plY3QtcGFja2FnaW5nQHZtd2FyZS5jb20+iQHSBBMBCAA8FiEEEIV//dP5Hq5X +eiHWZMu8gXPXaz8FAmPazmACGwMFCwkIBwIDIgIBBhUKCQgLAgQWAgMBAh4HAheA +AAoJEGTLvIFz12s/yf0L/jyP/LfduA4DwpjKX9Vpk26tgis9Q0I54UerpD5ibpTA +krzZxK1yFOPddcOjo+Xqg+I8aA+0nJkf+vsfnRgcpLs2qHZkikwZbPduZwkNUHX7 +6YPSXTwyFlzhaRycwPtvBPLFjfmjjjTi/aH4V/frfxfjH/wFvH/xiaiFsYbP3aAP +sJNTLh3im480ugQ7P54ukdte2QHKsjJ3z4tkjnu1ogc1+ZLCSZVDxfR4gLfE6GsN +YFNd+LF7+NtAeJRuJceXIisj8mTQYg+esTF9QtWovdg7vHVPz8mmcsrG9shGr+G9 +iwwtCig+hAGtXFAuODRMur9QfPlP6FhJw0FX/36iJ2p6APZB0EGqn7LJ91EyOnWv +iRimLLvlGFiVB9Xxw1TxnQMNj9jmB1CA4oNqlromO/AA0ryh13TpcIo5gbn6Jcdc +fD4Rbj5k+2HhJTkQ78GpZ0q95P08XD2dlaM2QxxKQGqADJOdV2VgjB2NDXURkInq +6pdkcaRgAKme8b+xjCcVjLkBjQRj2s5gAQwAxmgflHInM8oKQnsXezG5etLmaUsS +EkV5jjQFCShNn9zJEF/PWJk5Df/mbODj02wyc749dSJbRlTY3LgGz1AeywOsM1oQ +XkhfRZZqMwqvfx8IkEPjMvGIv/UI9pqqg/TY7OiYLEDahYXHJDKmlnmCBlnU96cL +yh7a/xY3ZC20/JwbFVAFzD4biWOrAm1YPpdKbqCPclpvRP9N6nb6hxvKKmDo7MqS +uANZMaoqhvnGazt9n435GQkYRvtqmqmOvt8I4oCzV0Y39HfbCHhhy64HSIowKYE7 +YWIujJcfoIDQqq2378T631BxLEUPaoSOV4B8gk/Jbf3KVu4LNqJive7chR8F1C2k +eeAKpaf2CSAe7OrbAfWysHRZ060bSJzRk3COEACk/UURY+RlIwh+LQxEKb1YQueS +YGjxIjV1X7ScyOvam5CmqOd4do9psOS7MHcQNeUbhnjm0TyGT9DF8ELoE0NSYa+J +PvDGHo51M33s31RUO4TtJnU5xSRb2sOKzIuBABEBAAGJAbYEGAEIACAWIQQQhX/9 +0/kerld6IdZky7yBc9drPwUCY9rOYAIbDAAKCRBky7yBc9drP8ctC/9wGi01cBAW +BPEKEnfrKdvlsaLeRxotriupDqGSWxqVxBVd+n0Xs0zPB/kuZFTkHOHpbAWkhPr+ +hP+RJemxCKMCo7kT2FXVR1OYej8Vh+aYWZ5lw6dJGtgo3Ebib2VSKdasmIOI2CY/ +03G46jv05qK3fP6phz+RaX+9hHgh1XW9kKbdkX5lM9RQSZOof3/67IN8w+euy61O +UhNcrsDKrp0kZxw3S+b/02oP1qADXHz2BUerkCZa4RVK1pM0UfRUooOHiEdUxKKM +DE501hwQsMH7WuvlIR8Oc2UGkEtzgukhmhpQPSsVPg54y9US+LkpztM+yq+zRu33 +gAfssli0MvSmkbcTDD22PGbgPMseyYxfw7vuwmjdqvi9Z4jdln2gyZ6sSZdgUMYW +PGEjZDoMzsZx9Zx6SO9XCS7XgYHVc8/B2LGSxj+rpZ6lBbywH88lNnrm/SpQB74U +4QVLffuw76FanTH6advqdWIqtlWPoAQcEkKf5CdmfT2ei2wX1QLatTs= +=ZKPF +-----END PGP PUBLIC KEY BLOCK----- diff --git a/salt/repo/client/files/rocky/keys/SALTSTACK-GPG-KEY2.pub b/salt/repo/client/files/rocky/keys/SALTSTACK-GPG-KEY2.pub deleted file mode 100644 index bfc7fc267..000000000 --- a/salt/repo/client/files/rocky/keys/SALTSTACK-GPG-KEY2.pub +++ /dev/null @@ -1,31 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- - -mQENBGLXV/8BCADCuomk2pibSOuLQeKMIwV3Afy60080hykdc4tU4qQS+zBJZZC0 -VBl2TAOmMWyeY5DRF2ibRTx6Ap8qYefuEjWlo2WHWWZH4WhNkJWL3aWiu8Ga+fFo -ebjoUFLGgpKDGKveO9PF8A41IP1CLvDicpWXTxfqzQKDOvg3g5EmCx+5ksviXHJ1 -lY5CBbhVPmU3ruzGBqN/6B90VyTicbIyIZKZdnElAqaW6OiEaOmj2Oadi3ARJLWA -8rpVPweZE0/S4B5UIuMh+JVJU3Os1BUXHKN3LAPENZa1NNYX3j53GxGMf+SAKe0g -QHe+fHiiB7a6iBl09W8cUJh8HINXW+vvU6mZABEBAAG0MlNhbHRTdGFjayBQYWNr -YWdpbmcgVGVhbSA8cGFja2FnaW5nQHNhbHRzdGFjay5jb20+iQFSBBMBCAA8FiEE -9+rekz4krjI0B2hWN6cQR50w17YFAmLXV/8CGwMFCwkIBwIDIgIBBhUKCQgLAgQW -AgMBAh4HAheAAAoJEDenEEedMNe2d0MH/36khQzCWMc5ezznO7bcOHOS3OWjQveF -Vv60y54QRnINCEa7w7ckjiap3dUSJxTo5eoAKNbgX5SgrshEY1HDXDoqgumHJLFW -J+L4f3CXFBhvObUOwB7ApUNHURcoNQYK7kS/vUJrQ3dFyT7uvgysGtv+/WpboY1s -ScJnVtWyQmLe7qj5pJ0aI5pPjFnP9869zPScNb6o6lbqGp/xhnL5NkZCF0DNgItw -HXyNsRPyc8JG+P+GP80XWZ37ajEdwkiPbtu3CD5pvBO1w5FPLBwuH5CSgQFEcA4V -QH8ThU0P1IhKe3xPRNgawcBTAHXqOD0OxilAIsQdfrKkRiTEcZtFZW25AQ0EYtdX -/wEIANFBzJfSks4ti/JQkECtEAwH7OtqUxu1QhSSRusGsQu/PpjBRZzlaVlKjS4c -fGTiZ8+25RX063vBQ+XpuTN9T9boEE4EywM11FCx1zRZIc+HlLOIJ10uKWUapmPM -+7flnQWXMgJzP47rHe0ofEHlP4/av5C1imgWEtEpYyn1B4qgSxvLFDq46rD5m+DP -2xNZbwWd0uSAG/wZNonVkISYymB0UTnUm8FABH1Ci7lXO9JnuW+IvVt32C5VibGy -FXdAJGmIiqsvBhJSUl+GJhO6NTXntuevqPLUXD9PuHWo4Vo1Afek8kqZByyiyrTZ -StDhrbo/8dSAVQMibLEfNS7R0QkAEQEAAYkBNgQYAQgAIBYhBPfq3pM+JK4yNAdo -VjenEEedMNe2BQJi11f/AhsMAAoJEDenEEedMNe2zhgH/0wxbQpaCho0BRbUbe6L -jm9r3yTWn6M+yYv+cBeH9sbobIVOqTvZcawzTEPWa+eVbKgkqhZjUTyfFDpjq9s6 -67zLZnCh85hLoyieSQBER59dc1pmqZJP3VrAIT1lGKMIdjZoN8JAF8IbmJHE1j65 -iZZdhbxfFHnDx22gQ+3nfniTNTWsfVAQeoAjeOuakPKdfUEMsXPBhtBBuFY4NcrT -TIsBevT4J/STCLkEqlMtYC8ldxUCZqQXdtxqltC4k+y0kp4PmNc3/Vmp65oAeuxI -d8TNwgZdamdinv5mPrTfBqSNiELQAcPQnOwpsqEDYF2pq9L4sdNGavP5ZvPGRLkH -+uU= -=383D ------END PGP PUBLIC KEY BLOCK----- diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index b4d80c476..dac0e6e5c 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -2,4 +2,4 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: master: - version: 3006.0+0na.61a7bd9 + version: 3006.0rc3 diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index 7bbd569c6..88a2435ca 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -2,6 +2,6 @@ # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions salt: minion: - version: 3006.0+0na.61a7bd9 + version: 3006.0rc3 check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default service_start_delay: 30 # in seconds. diff --git a/salt/setup/highstate_cron.sls b/salt/setup/highstate_cron.sls index dcbab4d8d..c14f4e1f4 100644 --- a/salt/setup/highstate_cron.sls +++ b/salt/setup/highstate_cron.sls @@ -1,6 +1,6 @@ post_setup_cron: cron.present: - - name: 'salt-call state.highstate' + - name: 'PATH=$PATH:/usr/sbin salt-call state.highstate' - user: root - minute: '*/1' - identifier: post_setup_cron diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 4a521f12c..9d280ff36 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -51,17 +51,13 @@ m2cryptopkgs: influxdb_key: x509.private_key_managed: - name: /etc/pki/influxdb.key - - CN: {{ GLOBALS.hostname }} - - bits: 4096 - - days_remaining: 0 - - days_valid: 820 + - keysize: 4096 - backup: True - new: True {% if salt['file.file_exists']('/etc/pki/influxdb.key') -%} - prereq: - x509: /etc/pki/influxdb.crt {%- endif %} - - timeout: 30 - retry: attempts: 5 interval: 30 @@ -72,7 +68,7 @@ influxdb_crt: - name: /etc/pki/influxdb.crt - ca_server: {{ ca_server }} - signing_policy: influxdb - - public_key: /etc/pki/influxdb.key + - private_key: /etc/pki/influxdb.key - CN: {{ GLOBALS.hostname }} - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - days_remaining: 0 @@ -101,17 +97,13 @@ influxkeyperms: redis_key: x509.private_key_managed: - name: /etc/pki/redis.key - - CN: {{ GLOBALS.hostname }} - - bits: 4096 - - days_remaining: 0 - - days_valid: 820 + - keysize: 4096 - backup: True - new: True {% if salt['file.file_exists']('/etc/pki/redis.key') -%} - prereq: - x509: /etc/pki/redis.crt {%- endif %} - - timeout: 30 - retry: attempts: 5 interval: 30 @@ -122,7 +114,7 @@ redis_crt: - ca_server: {{ ca_server }} - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - signing_policy: registry - - public_key: /etc/pki/redis.key + - private_key: /etc/pki/redis.key - CN: {{ GLOBALS.hostname }} - days_remaining: 0 - days_valid: 820 @@ -150,17 +142,13 @@ rediskeyperms: etc_elasticfleet_key: x509.private_key_managed: - name: /etc/pki/elasticfleet.key - - CN: {{ COMMONNAME }} - - bits: 4096 - - days_remaining: 0 - - days_valid: 820 + - keysize: 4096 - backup: True - new: True {% if salt['file.file_exists']('/etc/pki/elasticfleet.key') -%} - prereq: - x509: etc_elasticfleet_crt {%- endif %} - - timeout: 30 - retry: attempts: 5 interval: 30 @@ -171,7 +159,7 @@ etc_elasticfleet_crt: - name: /etc/pki/elasticfleet.crt - ca_server: {{ ca_server }} - signing_policy: elasticfleet - - public_key: /etc/pki/elasticfleet.key + - private_key: /etc/pki/elasticfleet.key - CN: {{ GLOBALS.hostname }} - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - days_remaining: 0 @@ -232,17 +220,13 @@ efcrtlink: etc_filebeat_key: x509.private_key_managed: - name: /etc/pki/filebeat.key - - CN: {{ COMMONNAME }} - - bits: 4096 - - days_remaining: 0 - - days_valid: 820 + - keysize: 4096 - backup: True - new: True {% if salt['file.file_exists']('/etc/pki/filebeat.key') -%} - prereq: - x509: etc_filebeat_crt {%- endif %} - - timeout: 30 - retry: attempts: 5 interval: 30 @@ -253,7 +237,7 @@ etc_filebeat_crt: - name: /etc/pki/filebeat.crt - ca_server: {{ ca_server }} - signing_policy: filebeat - - public_key: /etc/pki/filebeat.key + - private_key: /etc/pki/filebeat.key - CN: {{ GLOBALS.hostname }} - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - days_remaining: 0 @@ -313,17 +297,13 @@ fbcrtlink: registry_key: x509.private_key_managed: - name: /etc/pki/registry.key - - CN: {{ GLOBALS.manager }} - - bits: 4096 - - days_remaining: 0 - - days_valid: 820 + - keysize: 4096 - backup: True - new: True {% if salt['file.file_exists']('/etc/pki/registry.key') -%} - prereq: - x509: /etc/pki/registry.crt {%- endif %} - - timeout: 30 - retry: attempts: 5 interval: 30 @@ -335,7 +315,7 @@ registry_crt: - ca_server: {{ ca_server }} - subjectAltName: DNS:{{ GLOBALS.manager }}, IP:{{ GLOBALS.manager_ip }} - signing_policy: registry - - public_key: /etc/pki/registry.key + - private_key: /etc/pki/registry.key - CN: {{ GLOBALS.manager }} - days_remaining: 0 - days_valid: 820 @@ -361,17 +341,13 @@ regkeyperms: # Create a cert for elasticsearch /etc/pki/elasticsearch.key: x509.private_key_managed: - - CN: {{ COMMONNAME }} - - bits: 4096 - - days_remaining: 0 - - days_valid: 820 + - keysize: 4096 - backup: True - new: True {% if salt['file.file_exists']('/etc/pki/elasticsearch.key') -%} - prereq: - x509: /etc/pki/elasticsearch.crt {%- endif %} - - timeout: 30 - retry: attempts: 5 interval: 30 @@ -380,7 +356,7 @@ regkeyperms: x509.certificate_managed: - ca_server: {{ ca_server }} - signing_policy: registry - - public_key: /etc/pki/elasticsearch.key + - private_key: /etc/pki/elasticsearch.key - CN: {{ GLOBALS.hostname }} - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - days_remaining: 0 @@ -418,17 +394,13 @@ elasticp12perms: managerssl_key: x509.private_key_managed: - name: /etc/pki/managerssl.key - - CN: {{ GLOBALS.manager }} - - bits: 4096 - - days_remaining: 0 - - days_valid: 820 + - keysize: 4096 - backup: True - new: True {% if salt['file.file_exists']('/etc/pki/managerssl.key') -%} - prereq: - x509: /etc/pki/managerssl.crt {%- endif %} - - timeout: 30 - retry: attempts: 5 interval: 30 @@ -439,7 +411,7 @@ managerssl_crt: - name: /etc/pki/managerssl.crt - ca_server: {{ ca_server }} - signing_policy: managerssl - - public_key: /etc/pki/managerssl.key + - private_key: /etc/pki/managerssl.key - CN: {{ GLOBALS.hostname }} - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - days_remaining: 0 @@ -476,17 +448,13 @@ fbcertdir: conf_filebeat_key: x509.private_key_managed: - name: /opt/so/conf/filebeat/etc/pki/filebeat.key - - CN: {{ COMMONNAME }} - - bits: 4096 - - days_remaining: 0 - - days_valid: 820 + - keysize: 4096 - backup: True - new: True {% if salt['file.file_exists']('/opt/so/conf/filebeat/etc/pki/filebeat.key') -%} - prereq: - x509: conf_filebeat_crt {%- endif %} - - timeout: 30 - retry: attempts: 5 interval: 30 @@ -497,7 +465,7 @@ conf_filebeat_crt: - name: /opt/so/conf/filebeat/etc/pki/filebeat.crt - ca_server: {{ ca_server }} - signing_policy: filebeat - - public_key: /opt/so/conf/filebeat/etc/pki/filebeat.key + - private_key: /opt/so/conf/filebeat/etc/pki/filebeat.key - CN: {{ GLOBALS.hostname }} - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - days_remaining: 0 @@ -542,17 +510,13 @@ chownfilebeatp8: # Create a cert for elasticsearch /etc/pki/elasticsearch.key: x509.private_key_managed: - - CN: {{ GLOBALS.manager }} - - bits: 4096 - - days_remaining: 0 - - days_valid: 820 + - keysize: 4096 - backup: True - new: True {% if salt['file.file_exists']('/etc/pki/elasticsearch.key') -%} - prereq: - x509: /etc/pki/elasticsearch.crt {%- endif %} - - timeout: 30 - retry: attempts: 5 interval: 30 @@ -561,7 +525,7 @@ chownfilebeatp8: x509.certificate_managed: - ca_server: {{ ca_server }} - signing_policy: registry - - public_key: /etc/pki/elasticsearch.key + - private_key: /etc/pki/elasticsearch.key - CN: {{ GLOBALS.hostname }} - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - days_remaining: 0 diff --git a/setup/files/salt_module_deps/docker/certifi-2022.12.7-py3-none-any.whl b/setup/files/salt_module_deps/docker/certifi-2022.12.7-py3-none-any.whl new file mode 100644 index 000000000..a08305611 Binary files /dev/null and b/setup/files/salt_module_deps/docker/certifi-2022.12.7-py3-none-any.whl differ diff --git a/setup/files/salt_module_deps/docker/chardet-4.0.0-py2.py3-none-any.whl b/setup/files/salt_module_deps/docker/chardet-4.0.0-py2.py3-none-any.whl new file mode 100644 index 000000000..b83344e8e Binary files /dev/null and b/setup/files/salt_module_deps/docker/chardet-4.0.0-py2.py3-none-any.whl differ diff --git a/setup/files/salt_module_deps/docker/charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/setup/files/salt_module_deps/docker/charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 000000000..7b57bc716 Binary files /dev/null and b/setup/files/salt_module_deps/docker/charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/setup/files/salt_module_deps/docker/docker-5.0.2-py2.py3-none-any.whl b/setup/files/salt_module_deps/docker/docker-5.0.2-py2.py3-none-any.whl new file mode 100644 index 000000000..f1ed95ee1 Binary files /dev/null and b/setup/files/salt_module_deps/docker/docker-5.0.2-py2.py3-none-any.whl differ diff --git a/setup/files/salt_module_deps/docker/idna-2.10-py2.py3-none-any.whl b/setup/files/salt_module_deps/docker/idna-2.10-py2.py3-none-any.whl new file mode 100644 index 000000000..41225cb05 Binary files /dev/null and b/setup/files/salt_module_deps/docker/idna-2.10-py2.py3-none-any.whl differ diff --git a/setup/files/salt_module_deps/docker/requests-2.25.1-py2.py3-none-any.whl b/setup/files/salt_module_deps/docker/requests-2.25.1-py2.py3-none-any.whl new file mode 100644 index 000000000..8d70e9716 Binary files /dev/null and b/setup/files/salt_module_deps/docker/requests-2.25.1-py2.py3-none-any.whl differ diff --git a/setup/files/salt_module_deps/docker/urllib3-1.26.15-py2.py3-none-any.whl b/setup/files/salt_module_deps/docker/urllib3-1.26.15-py2.py3-none-any.whl new file mode 100644 index 000000000..ad723d5e1 Binary files /dev/null and b/setup/files/salt_module_deps/docker/urllib3-1.26.15-py2.py3-none-any.whl differ diff --git a/setup/files/salt_module_deps/docker/websocket_client-1.5.1-py3-none-any.whl b/setup/files/salt_module_deps/docker/websocket_client-1.5.1-py3-none-any.whl new file mode 100644 index 000000000..bd8152d2c Binary files /dev/null and b/setup/files/salt_module_deps/docker/websocket_client-1.5.1-py3-none-any.whl differ diff --git a/setup/files/salt_module_deps/pymysql/PyMySQL-1.0.3-py3-none-any.whl b/setup/files/salt_module_deps/pymysql/PyMySQL-1.0.3-py3-none-any.whl new file mode 100644 index 000000000..0bbd29da0 Binary files /dev/null and b/setup/files/salt_module_deps/pymysql/PyMySQL-1.0.3-py3-none-any.whl differ diff --git a/setup/so-functions b/setup/so-functions index a977ed142..9dbc95173 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -634,6 +634,8 @@ configure_minion() { printf '%s\n'\ "use_superseded:"\ " - module.run"\ + "features:"\ + " x509_v2: true"\ "log_level: info"\ "log_level_logfile: info"\ "log_file: /opt/so/log/salt/minion" >> "$minion_config" @@ -802,9 +804,11 @@ compare_main_nic_ip() { configure_network_sensor() { info "Setting up sensor interface" - if [[ $is_cloud ]]; then + if [[ $is_cloud ]]; then + info "Configuring traditional interface settings, since this is a cloud installation..." local nmcli_con_args=( "type" "ethernet" ) else + info "Configuring bond interface settings, since this is a not a cloud installation..." local nmcli_con_args=( "type" "bond" "mode" "0" ) fi @@ -909,7 +913,7 @@ create_repo() { detect_cloud() { info "Testing if setup is running on a cloud instance..." - if ( curl --fail -s -m 5 http://169.254.169.254/latest/meta-data/instance-id > /dev/null ) || ( dmidecode -s bios-vendor | grep -q Google > /dev/null) || [ -f /var/log/waagent.log ]; then export is_cloud="true"; fi + if ( curl --fail -s -m 5 http://169.254.169.254/latest/meta-data/instance-id > /dev/null ) || ( dmidecode -s bios-vendor | grep -q Google > /dev/null) || [ -f /var/log/waagent.log ]; then info "Detected a cloud installation..." && export is_cloud="true"; fi } detect_os() { @@ -1247,11 +1251,12 @@ generate_interface_vars() { export MTU # Set interface variable - if [[ $is_cloud ]]; then + if [[ $is_cloud ]]; then INTERFACE=${BNICS[0]} else INTERFACE='bond0' fi + info "Interface set to $INTERFACE" export INTERFACE } @@ -2026,8 +2031,11 @@ saltify() { if [[ $is_rocky ]]; then # THIS IS A TEMP HACK - logCmd "dnf -y install securityonion-salt python3-audit python3-libsemanage python3-policycoreutils python3-setools python3-setuptools python3-chardet python3-idna python3-pysocks python3-requests python3-urllib3 python3-websocket-client python3-docker" + #logCmd "dnf -y install securityonion-salt python3-audit python3-libsemanage python3-policycoreutils python3-setools python3-setuptools python3-chardet python3-idna python3-pysocks python3-requests python3-urllib3 python3-websocket-client python3-docker" + logCmd "dnf -y install salt salt-master salt-minion" logCmd "mkdir -p /etc/salt/minion.d" + logCmd "salt-pip install docker --no-index --only-binary=:all: --find-links files/salt_module_deps/docker/" + logCmd "salt-pip install pymysql --no-index --only-binary=:all: --find-links files/salt_module_deps/pymysql/" #if [[ $waitforstate ]]; then # # Since this is a salt master so let's install it # logCmd "" diff --git a/setup/so-setup b/setup/so-setup index f51ddb316..f10ff8c48 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -566,7 +566,7 @@ if ! [[ -f $install_opt_file ]]; then export NODE_DESCRIPTION=$NODE_DESCRIPTION export MAINIP=$MAINIP export PATCHSCHEDULENAME=$PATCHSCHEDULENAME - export INTERFACE="bond0" + export INTERFACE=$INTERFACE export CORECOUNT=$lb_procs export LSHOSTNAME=$HOSTNAME export LSHEAP=$LS_HEAP_SIZE