mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-01-06 00:03:10 +01:00
merge 2.4/dev
Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>
This commit is contained in:
reyesj2
@@ -97,8 +97,6 @@ function soUserSync() {
|
||||
salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply logstash queue=True
|
||||
printf "\nApplying kibana state to the appropriate nodes.\n\n"
|
||||
salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch' state.apply kibana queue=True
|
||||
printf "\nApplying curator state to the appropriate nodes.\n\n"
|
||||
salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply curator queue=True
|
||||
}
|
||||
|
||||
function highstateManager() {
|
||||
|
||||
@@ -9,23 +9,23 @@ if [ -f /usr/sbin/so-common ]; then
|
||||
. /usr/sbin/so-common
|
||||
fi
|
||||
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
echo "This script must be run using sudo!"
|
||||
function usage() {
|
||||
echo "Usage: $0 -o=<operation> -m=[id]"
|
||||
echo ""
|
||||
echo " where <operation> is one of the following:"
|
||||
echo ""
|
||||
echo " add: Accepts a new key and adds the minion files"
|
||||
echo " delete: Removes the key and deletes the minion files"
|
||||
echo " list: Lists all keys with hashes"
|
||||
echo " reject: Rejects a key"
|
||||
echo " restart: Restart a minion (reboot)"
|
||||
echo " test: Perform minion test"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ $# -lt 1 ]]; then
|
||||
echo "Usage: $0 -o=<operation> -m=[id]"
|
||||
echo ""
|
||||
echo " where <operation> is one of the following:"
|
||||
echo ""
|
||||
echo " list: Lists all keys with hashes"
|
||||
echo " accept: Accepts a new key and adds the minion files"
|
||||
echo " delete: Removes the key and deletes the minion files"
|
||||
echo " reject: Rejects a key"
|
||||
echo " test: Perform minion test"
|
||||
echo ""
|
||||
exit 1
|
||||
usage
|
||||
fi
|
||||
|
||||
for i in "$@"; do
|
||||
@@ -38,6 +38,8 @@ for i in "$@"; do
|
||||
MINION_ID="${i#*=}"
|
||||
shift
|
||||
;;
|
||||
|
||||
# The following args are used internally during setup, not to be specified manually.
|
||||
-e=*|--esheap=*)
|
||||
ES_HEAP_SIZE="${i#*=}"
|
||||
shift
|
||||
@@ -63,6 +65,7 @@ for i in "$@"; do
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
@@ -76,7 +79,33 @@ function getinstallinfo() {
|
||||
source <(echo $INSTALLVARS)
|
||||
}
|
||||
|
||||
function testminion() {
|
||||
function pcapspace() {
|
||||
if [[ "$OPERATION" == "setup" ]]; then
|
||||
# Use 25% for PCAP
|
||||
PCAP_PERCENTAGE=1
|
||||
DFREEPERCENT=21
|
||||
local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n)
|
||||
else
|
||||
|
||||
local NSMSIZE=$(salt "$MINION_ID" disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ')
|
||||
local ROOTSIZE=$(salt "$MINION_ID" disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ')
|
||||
|
||||
if [[ "$NSMSIZE" == "null" ]]; then
|
||||
# Looks like there is no dedicated nsm partition. Using root
|
||||
local SPACESIZE=$ROOTSIZE
|
||||
else
|
||||
local SPACESIZE=$NSMSIZE
|
||||
fi
|
||||
fi
|
||||
|
||||
local s=$(( $SPACESIZE / 1000000 ))
|
||||
local s1=$(( $s / 4 * $PCAP_PERCENTAGE ))
|
||||
|
||||
MAX_PCAP_SPACE=$s1
|
||||
|
||||
}
|
||||
|
||||
function testMinion() {
|
||||
# Always run on the host, since this is going to be the manager of a distributed grid, or an eval/standalone.
|
||||
# Distributed managers must run this in order for the sensor nodes to have access to the so-tcpreplay image.
|
||||
so-test
|
||||
@@ -92,12 +121,19 @@ function testminion() {
|
||||
exit $result
|
||||
}
|
||||
|
||||
function listminions() {
|
||||
function restartMinion() {
|
||||
salt "$MINION_ID" system.reboot
|
||||
result=$?
|
||||
|
||||
exit $result
|
||||
}
|
||||
|
||||
function listMinions() {
|
||||
salt-key list -F --out=json
|
||||
exit $?
|
||||
}
|
||||
|
||||
function rejectminion() {
|
||||
function rejectMinion() {
|
||||
salt-key -y -r $MINION_ID
|
||||
exit $?
|
||||
}
|
||||
@@ -106,11 +142,11 @@ function acceptminion() {
|
||||
salt-key -y -a $MINION_ID
|
||||
}
|
||||
|
||||
function deleteminion() {
|
||||
function deleteMinion() {
|
||||
salt-key -y -d $MINION_ID
|
||||
}
|
||||
|
||||
function deleteminionfiles () {
|
||||
function deleteMinionFiles () {
|
||||
rm -f $PILLARFILE
|
||||
rm -f $ADVPILLARFILE
|
||||
}
|
||||
@@ -234,6 +270,10 @@ function add_sensor_to_minion() {
|
||||
echo " lb_procs: '$CORECOUNT'" >> $PILLARFILE
|
||||
echo "suricata:" >> $PILLARFILE
|
||||
echo " enabled: True " >> $PILLARFILE
|
||||
if [[ $is_pcaplimit ]]; then
|
||||
echo " pcap:" >> $PILLARFILE
|
||||
echo " maxsize: $MAX_PCAP_SPACE" >> $PILLARFILE
|
||||
fi
|
||||
echo " config:" >> $PILLARFILE
|
||||
echo " af-packet:" >> $PILLARFILE
|
||||
echo " threads: '$CORECOUNT'" >> $PILLARFILE
|
||||
@@ -241,7 +281,7 @@ function add_sensor_to_minion() {
|
||||
echo " enabled: True" >> $PILLARFILE
|
||||
if [[ $is_pcaplimit ]]; then
|
||||
echo " config:" >> $PILLARFILE
|
||||
echo " diskfreepercentage: 60" >> $PILLARFILE
|
||||
echo " diskfreepercentage: $DFREEPERCENT" >> $PILLARFILE
|
||||
fi
|
||||
echo " " >> $PILLARFILE
|
||||
}
|
||||
@@ -292,13 +332,6 @@ function add_strelka_to_minion() {
|
||||
" " >> $PILLARFILE
|
||||
}
|
||||
|
||||
function add_curator_to_minion() {
|
||||
printf '%s\n'\
|
||||
"curator:"\
|
||||
" enabled: True"\
|
||||
" " >> $PILLARFILE
|
||||
}
|
||||
|
||||
function add_telegraf_to_minion() {
|
||||
printf '%s\n'\
|
||||
"telegraf:"\
|
||||
@@ -419,13 +452,13 @@ function updateMine() {
|
||||
|
||||
function createEVAL() {
|
||||
is_pcaplimit=true
|
||||
pcapspace
|
||||
add_elasticsearch_to_minion
|
||||
add_sensor_to_minion
|
||||
add_strelka_to_minion
|
||||
add_playbook_to_minion
|
||||
add_elastalert_to_minion
|
||||
add_kibana_to_minion
|
||||
add_curator_to_minion
|
||||
add_telegraf_to_minion
|
||||
add_influxdb_to_minion
|
||||
add_nginx_to_minion
|
||||
@@ -440,6 +473,7 @@ function createEVAL() {
|
||||
|
||||
function createSTANDALONE() {
|
||||
is_pcaplimit=true
|
||||
pcapspace
|
||||
add_elasticsearch_to_minion
|
||||
add_logstash_to_minion
|
||||
add_sensor_to_minion
|
||||
@@ -448,7 +482,6 @@ function createSTANDALONE() {
|
||||
add_elastalert_to_minion
|
||||
add_kibana_to_minion
|
||||
add_redis_to_minion
|
||||
add_curator_to_minion
|
||||
add_telegraf_to_minion
|
||||
add_influxdb_to_minion
|
||||
add_nginx_to_minion
|
||||
@@ -468,7 +501,6 @@ function createMANAGER() {
|
||||
add_elastalert_to_minion
|
||||
add_kibana_to_minion
|
||||
add_redis_to_minion
|
||||
add_curator_to_minion
|
||||
add_telegraf_to_minion
|
||||
add_influxdb_to_minion
|
||||
add_nginx_to_minion
|
||||
@@ -488,7 +520,6 @@ function createMANAGERSEARCH() {
|
||||
add_elastalert_to_minion
|
||||
add_kibana_to_minion
|
||||
add_redis_to_minion
|
||||
add_curator_to_minion
|
||||
add_telegraf_to_minion
|
||||
add_influxdb_to_minion
|
||||
add_nginx_to_minion
|
||||
@@ -532,17 +563,23 @@ function createIDH() {
|
||||
|
||||
function createHEAVYNODE() {
|
||||
is_pcaplimit=true
|
||||
PCAP_PERCENTAGE=1
|
||||
DFREEPERCENT=21
|
||||
pcapspace
|
||||
add_elasticsearch_to_minion
|
||||
add_elastic_agent_to_minion
|
||||
add_logstash_to_minion
|
||||
add_sensor_to_minion
|
||||
add_strelka_to_minion
|
||||
add_redis_to_minion
|
||||
add_curator_to_minion
|
||||
add_telegraf_to_minion
|
||||
}
|
||||
|
||||
function createSENSOR() {
|
||||
is_pcaplimit=true
|
||||
DFREEPERCENT=10
|
||||
PCAP_PERCENTAGE=3
|
||||
pcapspace
|
||||
add_sensor_to_minion
|
||||
add_strelka_to_minion
|
||||
add_telegraf_to_minion
|
||||
@@ -594,25 +631,33 @@ function testConnection() {
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ "$OPERATION" = 'list' ]]; then
|
||||
listminions
|
||||
fi
|
||||
function addMinion() {
|
||||
# Accept the salt key
|
||||
acceptminion
|
||||
# Test to see if the minion was accepted
|
||||
testConnection
|
||||
# Pull the info from the file to build what is needed
|
||||
getinstallinfo
|
||||
}
|
||||
|
||||
if [[ "$OPERATION" = 'delete' ]]; then
|
||||
deleteminionfiles
|
||||
deleteminion
|
||||
fi
|
||||
|
||||
if [[ "$OPERATION" == 'add' || "$OPERATION" == 'setup' ]]; then
|
||||
# Skip this if its setup
|
||||
if [[ $OPERATION == 'add' ]]; then
|
||||
# Accept the salt key
|
||||
acceptminion
|
||||
# Test to see if the minion was accepted
|
||||
testConnection
|
||||
# Pull the info from the file to build what is needed
|
||||
getinstallinfo
|
||||
function updateMineAndApplyStates() {
|
||||
# tell the minion to populate the mine with data from mine_functions which is populated during setup
|
||||
# this only needs to happen on non managers since they handle this during setup
|
||||
# and they need to wait for ca creation to update the mine
|
||||
updateMine
|
||||
checkMine "network.ip_addrs"
|
||||
# apply the elasticsearch state to the manager if a new searchnode was added
|
||||
if [[ "$NODETYPE" == "SEARCHNODE" || "$NODETYPE" == "HEAVYNODE" ]]; then
|
||||
# calls so-common and set_minionid sets MINIONID to local minion id
|
||||
set_minionid
|
||||
salt $MINIONID state.apply elasticsearch queue=True --async
|
||||
salt $MINIONID state.apply soc queue=True --async
|
||||
fi
|
||||
# run this async so the cli doesn't wait for a return
|
||||
salt "$MINION_ID" state.highstate --async queue=True
|
||||
}
|
||||
|
||||
function setupMinionFiles() {
|
||||
# Check to see if nodetype is set
|
||||
if [ -z $NODETYPE ]; then
|
||||
echo "No node type specified"
|
||||
@@ -629,25 +674,41 @@ if [[ "$OPERATION" == 'add' || "$OPERATION" == 'setup' ]]; then
|
||||
|
||||
create$NODETYPE
|
||||
echo "Minion file created for $MINION_ID"
|
||||
}
|
||||
|
||||
if [[ "$OPERATION" == 'add' ]]; then
|
||||
# tell the minion to populate the mine with data from mine_functions which is populated during setup
|
||||
# this only needs to happen on non managers since they handle this during setup
|
||||
# and they need to wait for ca creation to update the mine
|
||||
updateMine
|
||||
checkMine "network.ip_addrs"
|
||||
# apply the elasticsearch state to the manager if a new searchnode was added
|
||||
if [[ "$NODETYPE" == "SEARCHNODE" || "$NODETYPE" == "HEAVYNODE" ]]; then
|
||||
# calls so-common and set_minionid sets MINIONID to local minion id
|
||||
set_minionid
|
||||
salt $MINIONID state.apply elasticsearch queue=True --async
|
||||
salt $MINIONID state.apply soc queue=True --async
|
||||
fi
|
||||
# run this async so the cli doesn't wait for a return
|
||||
salt "$MINION_ID" state.highstate --async queue=True
|
||||
fi
|
||||
fi
|
||||
case "$OPERATION" in
|
||||
"add")
|
||||
addMinion
|
||||
setupMinionFiles
|
||||
updateMineAndApplyStates
|
||||
;;
|
||||
|
||||
if [[ "$OPERATION" = 'test' ]]; then
|
||||
testminion
|
||||
fi
|
||||
"delete")
|
||||
deleteMinionFiles
|
||||
deleteMinion
|
||||
;;
|
||||
|
||||
"list")
|
||||
listMinions
|
||||
;;
|
||||
|
||||
"reject")
|
||||
rejectMinion
|
||||
;;
|
||||
|
||||
"restart")
|
||||
restartMinion
|
||||
;;
|
||||
|
||||
"setup")
|
||||
# only should be invoked directly during setup, never manually
|
||||
setupMinionFiles
|
||||
;;
|
||||
|
||||
"test")
|
||||
testMinion
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -7,12 +7,8 @@
|
||||
NOROOT=1
|
||||
. /usr/sbin/so-common
|
||||
|
||||
set_version
|
||||
set_os
|
||||
salt_minion_count
|
||||
|
||||
set -e
|
||||
|
||||
curl --retry 5 --retry-delay 60 -A "reposync/$VERSION/$OS/$(uname -r)/$MINIONCOUNT" https://sigs.securityonion.net/checkup --output /tmp/checkup
|
||||
curl --retry 5 --retry-delay 60 -A "reposync/$(sync_options)" https://sigs.securityonion.net/checkup --output /tmp/checkup
|
||||
dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/
|
||||
createrepo /nsm/repo
|
||||
|
||||
@@ -47,7 +47,7 @@ got_root(){
|
||||
|
||||
got_root
|
||||
if [ $# -ne 1 ] ; then
|
||||
BRANCH=master
|
||||
BRANCH=2.4/main
|
||||
else
|
||||
BRANCH=$1
|
||||
fi
|
||||
|
||||
@@ -347,7 +347,7 @@ function syncElastic() {
|
||||
[[ $? != 0 ]] && fail "Unable to read credential hashes from database"
|
||||
|
||||
user_data_formatted=$(echo "${userData}" | jq -r '.user + ":" + .data.hashed_password')
|
||||
if lookup_salt_value "licensed_features" "" "pillar" | grep -x oidc; then
|
||||
if lookup_salt_value "features" "" "pillar" | grep -x odc; then
|
||||
# generate random placeholder salt/hash for users without passwords
|
||||
random_crypt=$(get_random_value 53)
|
||||
user_data_formatted=$(echo "${user_data_formatted}" | sed -r "s/^(.+:)\$/\\1\$2a\$12${random_crypt}/")
|
||||
|
||||
@@ -16,12 +16,14 @@ lockFile = "/tmp/so-yaml.lock"
|
||||
def showUsage(args):
|
||||
print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]))
|
||||
print(' General commands:')
|
||||
print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.')
|
||||
print(' remove - Removes a yaml key, if it exists. Requires KEY arg.')
|
||||
print(' help - Prints this usage information.')
|
||||
print('')
|
||||
print(' Where:')
|
||||
print(' YAML_FILE - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml')
|
||||
print(' KEY - YAML key, does not support \' or " characters at this time. Ex: level1.level2')
|
||||
print(' LISTITEM - Item to add to the list.')
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@@ -35,6 +37,35 @@ def writeYaml(filename, content):
|
||||
file = open(filename, "w")
|
||||
return yaml.dump(content, file)
|
||||
|
||||
def appendItem(content, key, listItem):
|
||||
pieces = key.split(".", 1)
|
||||
if len(pieces) > 1:
|
||||
appendItem(content[pieces[0]], pieces[1], listItem)
|
||||
else:
|
||||
try:
|
||||
content[key].append(listItem)
|
||||
except AttributeError:
|
||||
print("The existing value for the given key is not a list. No action was taken on the file.")
|
||||
return 1
|
||||
except KeyError:
|
||||
print("The key provided does not exist. No action was taken on the file.")
|
||||
return 1
|
||||
|
||||
def append(args):
|
||||
if len(args) != 3:
|
||||
print('Missing filename, key arg, or list item to append', file=sys.stderr)
|
||||
showUsage(None)
|
||||
return
|
||||
|
||||
filename = args[0]
|
||||
key = args[1]
|
||||
listItem = args[2]
|
||||
|
||||
content = loadYaml(filename)
|
||||
appendItem(content, key, listItem)
|
||||
writeYaml(filename, content)
|
||||
|
||||
return 0
|
||||
|
||||
def removeKey(content, key):
|
||||
pieces = key.split(".", 1)
|
||||
@@ -43,6 +74,7 @@ def removeKey(content, key):
|
||||
else:
|
||||
content.pop(key, None)
|
||||
|
||||
|
||||
def remove(args):
|
||||
if len(args) != 2:
|
||||
print('Missing filename or key arg', file=sys.stderr)
|
||||
@@ -68,6 +100,7 @@ def main():
|
||||
|
||||
commands = {
|
||||
"help": showUsage,
|
||||
"append": append,
|
||||
"remove": remove,
|
||||
}
|
||||
|
||||
|
||||
@@ -105,3 +105,99 @@ class TestRemove(unittest.TestCase):
|
||||
self.assertEqual(actual, expected)
|
||||
sysmock.assert_called_once_with(1)
|
||||
self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n")
|
||||
|
||||
def test_append(self):
|
||||
filename = "/tmp/so-yaml_test-remove.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}")
|
||||
file.close()
|
||||
|
||||
soyaml.append([filename, "key3", "d"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
expected = "key1:\n child1: 123\n child2: abc\nkey2: false\nkey3:\n- a\n- b\n- c\n- d\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_append_nested(self):
|
||||
filename = "/tmp/so-yaml_test-remove.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}")
|
||||
file.close()
|
||||
|
||||
soyaml.append([filename, "key1.child2", "d"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n child1: 123\n child2:\n - a\n - b\n - c\n - d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_append_nested_deep(self):
|
||||
filename = "/tmp/so-yaml_test-remove.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
|
||||
file.close()
|
||||
|
||||
soyaml.append([filename, "key1.child2.deep2", "d"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n child1: 123\n child2:\n deep1: 45\n deep2:\n - a\n - b\n - c\n - d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_append_key_noexist(self):
|
||||
filename = "/tmp/so-yaml_test-append.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
|
||||
file.close()
|
||||
|
||||
with patch('sys.exit', new=MagicMock()) as sysmock:
|
||||
with patch('sys.stdout', new=StringIO()) as mock_stdout:
|
||||
sys.argv = ["cmd", "append", filename, "key4", "h"]
|
||||
soyaml.main()
|
||||
sysmock.assert_called()
|
||||
self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n")
|
||||
|
||||
def test_append_key_noexist_deep(self):
|
||||
filename = "/tmp/so-yaml_test-append.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
|
||||
file.close()
|
||||
|
||||
with patch('sys.exit', new=MagicMock()) as sysmock:
|
||||
with patch('sys.stdout', new=StringIO()) as mock_stdout:
|
||||
sys.argv = ["cmd", "append", filename, "key1.child2.deep3", "h"]
|
||||
soyaml.main()
|
||||
sysmock.assert_called()
|
||||
self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n")
|
||||
|
||||
def test_append_key_nonlist(self):
|
||||
filename = "/tmp/so-yaml_test-append.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
|
||||
file.close()
|
||||
|
||||
with patch('sys.exit', new=MagicMock()) as sysmock:
|
||||
with patch('sys.stdout', new=StringIO()) as mock_stdout:
|
||||
sys.argv = ["cmd", "append", filename, "key1", "h"]
|
||||
soyaml.main()
|
||||
sysmock.assert_called()
|
||||
self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n")
|
||||
|
||||
def test_append_key_nonlist_deep(self):
|
||||
filename = "/tmp/so-yaml_test-append.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
|
||||
file.close()
|
||||
|
||||
with patch('sys.exit', new=MagicMock()) as sysmock:
|
||||
with patch('sys.stdout', new=StringIO()) as mock_stdout:
|
||||
sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"]
|
||||
soyaml.main()
|
||||
sysmock.assert_called()
|
||||
self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n")
|
||||
|
||||
@@ -247,67 +247,6 @@ check_sudoers() {
|
||||
fi
|
||||
}
|
||||
|
||||
check_log_size_limit() {
|
||||
local num_minion_pillars
|
||||
num_minion_pillars=$(find /opt/so/saltstack/local/pillar/minions/ -type f | wc -l)
|
||||
|
||||
if [[ $num_minion_pillars -gt 1 ]]; then
|
||||
if find /opt/so/saltstack/local/pillar/minions/ -type f | grep -q "_heavynode"; then
|
||||
lsl_msg='distributed'
|
||||
fi
|
||||
else
|
||||
local minion_id
|
||||
minion_id=$(lookup_salt_value "id" "" "grains" "" "local")
|
||||
|
||||
local minion_arr
|
||||
IFS='_' read -ra minion_arr <<< "$minion_id"
|
||||
|
||||
local node_type="${minion_arr[0]}"
|
||||
|
||||
local current_limit
|
||||
# since it is possible for the salt-master service to be stopped when this is run, we need to check the pillar values locally
|
||||
# we need to combine default local and default pillars before doing this so we can define --pillar-root in salt-call
|
||||
local epoch_date=$(date +%s%N)
|
||||
mkdir -vp /opt/so/saltstack/soup_tmp_${epoch_date}/
|
||||
cp -r /opt/so/saltstack/default/pillar/ /opt/so/saltstack/soup_tmp_${epoch_date}/
|
||||
# use \cp here to overwrite any pillar files from default with those in local for the tmp directory
|
||||
\cp -r /opt/so/saltstack/local/pillar/ /opt/so/saltstack/soup_tmp_${epoch_date}/
|
||||
current_limit=$(salt-call pillar.get elasticsearch:log_size_limit --local --pillar-root=/opt/so/saltstack/soup_tmp_${epoch_date}/pillar --out=newline_values_only)
|
||||
rm -rf /opt/so/saltstack/soup_tmp_${epoch_date}/
|
||||
|
||||
local percent
|
||||
case $node_type in
|
||||
'standalone' | 'eval')
|
||||
percent=50
|
||||
;;
|
||||
*)
|
||||
percent=80
|
||||
;;
|
||||
esac
|
||||
|
||||
local disk_dir="/"
|
||||
if [ -d /nsm ]; then
|
||||
disk_dir="/nsm"
|
||||
fi
|
||||
|
||||
local disk_size_1k
|
||||
disk_size_1k=$(df $disk_dir | grep -v "^Filesystem" | awk '{print $2}')
|
||||
|
||||
local ratio="1048576"
|
||||
|
||||
local disk_size_gb
|
||||
disk_size_gb=$( echo "$disk_size_1k" "$ratio" | awk '{print($1/$2)}' )
|
||||
|
||||
local new_limit
|
||||
new_limit=$( echo "$disk_size_gb" "$percent" | awk '{printf("%.0f", $1 * ($2/100))}')
|
||||
|
||||
if [[ $current_limit != "$new_limit" ]]; then
|
||||
lsl_msg='single-node'
|
||||
lsl_details=( "$current_limit" "$new_limit" "$minion_id" )
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
check_os_updates() {
|
||||
# Check to see if there are OS updates
|
||||
echo "Checking for OS updates."
|
||||
@@ -372,6 +311,17 @@ enable_highstate() {
|
||||
echo ""
|
||||
}
|
||||
|
||||
get_soup_script_hashes() {
|
||||
CURRENTSOUP=$(md5sum /usr/sbin/soup | awk '{print $1}')
|
||||
GITSOUP=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/soup | awk '{print $1}')
|
||||
CURRENTCMN=$(md5sum /usr/sbin/so-common | awk '{print $1}')
|
||||
GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}')
|
||||
CURRENTIMGCMN=$(md5sum /usr/sbin/so-image-common | awk '{print $1}')
|
||||
GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}')
|
||||
CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}')
|
||||
GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-firewall | awk '{print $1}')
|
||||
}
|
||||
|
||||
highstate() {
|
||||
# Run a highstate.
|
||||
salt-call state.highstate -l info queue=True
|
||||
@@ -404,6 +354,9 @@ preupgrade_changes() {
|
||||
[[ "$INSTALLEDVERSION" == 2.4.5 ]] && up_to_2.4.10
|
||||
[[ "$INSTALLEDVERSION" == 2.4.10 ]] && up_to_2.4.20
|
||||
[[ "$INSTALLEDVERSION" == 2.4.20 ]] && up_to_2.4.30
|
||||
[[ "$INSTALLEDVERSION" == 2.4.30 ]] && up_to_2.4.40
|
||||
[[ "$INSTALLEDVERSION" == 2.4.40 ]] && up_to_2.4.50
|
||||
[[ "$INSTALLEDVERSION" == 2.4.50 ]] && up_to_2.4.60
|
||||
true
|
||||
}
|
||||
|
||||
@@ -416,7 +369,10 @@ postupgrade_changes() {
|
||||
[[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5
|
||||
[[ "$POSTVERSION" == 2.4.5 ]] && post_to_2.4.10
|
||||
[[ "$POSTVERSION" == 2.4.10 ]] && post_to_2.4.20
|
||||
[[ "$POSTVERSION" == 2.4.20 ]] && post_to_2.4.30
|
||||
[[ "$POSTVERSION" == 2.4.20 ]] && post_to_2.4.30
|
||||
[[ "$POSTVERSION" == 2.4.30 ]] && post_to_2.4.40
|
||||
[[ "$POSTVERSION" == 2.4.40 ]] && post_to_2.4.50
|
||||
[[ "$POSTVERSION" == 2.4.50 ]] && post_to_2.4.60
|
||||
true
|
||||
}
|
||||
|
||||
@@ -450,7 +406,10 @@ post_to_2.4.20() {
|
||||
post_to_2.4.30() {
|
||||
echo "Regenerating Elastic Agent Installers"
|
||||
/sbin/so-elastic-agent-gen-installers
|
||||
# there is an occasional error with this state: pki_public_ca_crt: TypeError: list indices must be integers or slices, not str
|
||||
set +e
|
||||
salt-call state.apply ca queue=True
|
||||
set -e
|
||||
stop_salt_minion
|
||||
mv /etc/pki/managerssl.crt /etc/pki/managerssl.crt.old
|
||||
mv /etc/pki/managerssl.key /etc/pki/managerssl.key.old
|
||||
@@ -460,6 +419,22 @@ post_to_2.4.30() {
|
||||
POSTVERSION=2.4.30
|
||||
}
|
||||
|
||||
post_to_2.4.40() {
|
||||
echo "Nothing to apply"
|
||||
POSTVERSION=2.4.40
|
||||
}
|
||||
|
||||
post_to_2.4.50() {
|
||||
echo "Nothing to apply"
|
||||
POSTVERSION=2.4.50
|
||||
}
|
||||
|
||||
post_to_2.4.60() {
|
||||
echo "Regenerating Elastic Agent Installers..."
|
||||
so-elastic-agent-gen-installers
|
||||
POSTVERSION=2.4.60
|
||||
}
|
||||
|
||||
repo_sync() {
|
||||
echo "Sync the local repo."
|
||||
su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
|
||||
@@ -552,6 +527,53 @@ up_to_2.4.30() {
|
||||
INSTALLEDVERSION=2.4.30
|
||||
}
|
||||
|
||||
up_to_2.4.40() {
|
||||
echo "Removing old ATT&CK Navigator Layers..."
|
||||
rm -f /opt/so/conf/navigator/layers/enterprise-attack.json
|
||||
rm -f /opt/so/conf/navigator/layers/nav_layer_playbook.json
|
||||
|
||||
INSTALLEDVERSION=2.4.40
|
||||
}
|
||||
|
||||
up_to_2.4.50() {
|
||||
echo "Creating additional pillars.."
|
||||
mkdir -p /opt/so/saltstack/local/pillar/stig/
|
||||
mkdir -p /opt/so/saltstack/local/salt/stig/
|
||||
chown socore:socore /opt/so/saltstack/local/salt/stig/
|
||||
touch /opt/so/saltstack/local/pillar/stig/adv_stig.sls
|
||||
touch /opt/so/saltstack/local/pillar/stig/soc_stig.sls
|
||||
|
||||
# the file_roots need to be update due to salt 3006.6 upgrade not allowing symlinks outside the file_roots
|
||||
# put new so-yaml in place
|
||||
echo "Updating so-yaml"
|
||||
\cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" "$DEFAULT_SALT_DIR/salt/manager/tools/sbin/"
|
||||
\cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" /usr/sbin/
|
||||
echo "Creating a backup of the salt-master config."
|
||||
# INSTALLEDVERSION is 2.4.40 at this point, but we want the backup to have the version
|
||||
# so was at prior to starting upgrade. use POSTVERSION here since it doesnt change until
|
||||
# post upgrade changes. POSTVERSION set to INSTALLEDVERSION at start of soup
|
||||
cp -v /etc/salt/master "/etc/salt/master.so-$POSTVERSION.bak"
|
||||
echo "Adding /opt/so/rules to file_roots in /etc/salt/master using so-yaml"
|
||||
so-yaml.py append /etc/salt/master file_roots.base /opt/so/rules/nids
|
||||
echo "Moving Suricata rules"
|
||||
mkdir /opt/so/rules/nids/suri
|
||||
chown socore:socore /opt/so/rules/nids/suri
|
||||
mv -v /opt/so/rules/nids/*.rules /opt/so/rules/nids/suri/.
|
||||
|
||||
echo "Adding /nsm/elastic-fleet/artifacts to file_roots in /etc/salt/master using so-yaml"
|
||||
so-yaml.py append /etc/salt/master file_roots.base /nsm/elastic-fleet/artifacts
|
||||
|
||||
INSTALLEDVERSION=2.4.50
|
||||
}
|
||||
|
||||
up_to_2.4.60() {
|
||||
echo "Creating directory to store Suricata classification.config"
|
||||
mkdir -vp /opt/so/saltstack/local/salt/suricata/classification
|
||||
chown socore:socore /opt/so/saltstack/local/salt/suricata/classification
|
||||
|
||||
INSTALLEDVERSION=2.4.60
|
||||
}
|
||||
|
||||
determine_elastic_agent_upgrade() {
|
||||
if [[ $is_airgap -eq 0 ]]; then
|
||||
update_elastic_agent_airgap
|
||||
@@ -594,7 +616,15 @@ unmount_update() {
|
||||
|
||||
update_airgap_rules() {
|
||||
# Copy the rules over to update them for airgap.
|
||||
rsync -av $UPDATE_DIR/agrules/* /nsm/repo/rules/
|
||||
rsync -av $UPDATE_DIR/agrules/suricata/* /nsm/rules/suricata/
|
||||
rsync -av $UPDATE_DIR/agrules/yara/* /nsm/rules/yara/
|
||||
if [ -d /nsm/repo/rules/sigma ]; then
|
||||
rsync -av $UPDATE_DIR/agrules/sigma/* /nsm/repo/rules/sigma/
|
||||
fi
|
||||
|
||||
# SOC Detections Airgap
|
||||
rsync -av $UPDATE_DIR/agrules/detect-sigma/* /nsm/rules/detect-sigma/
|
||||
rsync -av $UPDATE_DIR/agrules/detect-yara/* /nsm/rules/detect-yara/
|
||||
}
|
||||
|
||||
update_airgap_repo() {
|
||||
@@ -720,31 +750,29 @@ upgrade_salt() {
|
||||
}
|
||||
|
||||
verify_latest_update_script() {
|
||||
# Check to see if the update scripts match. If not run the new one.
|
||||
CURRENTSOUP=$(md5sum /usr/sbin/soup | awk '{print $1}')
|
||||
GITSOUP=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/soup | awk '{print $1}')
|
||||
CURRENTCMN=$(md5sum /usr/sbin/so-common | awk '{print $1}')
|
||||
GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}')
|
||||
CURRENTIMGCMN=$(md5sum /usr/sbin/so-image-common | awk '{print $1}')
|
||||
GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}')
|
||||
CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}')
|
||||
GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-firewall | awk '{print $1}')
|
||||
|
||||
get_soup_script_hashes
|
||||
if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
|
||||
echo "This version of the soup script is up to date. Proceeding."
|
||||
else
|
||||
echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete."
|
||||
cp $UPDATE_DIR/salt/manager/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/
|
||||
cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
|
||||
cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
|
||||
cp $UPDATE_DIR/salt/manager/tools/sbin/so-firewall $DEFAULT_SALT_DIR/salt/common/tools/sbin/
|
||||
|
||||
salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local
|
||||
|
||||
# Verify that soup scripts updated as expected
|
||||
get_soup_script_hashes
|
||||
if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
|
||||
echo "Succesfully updated soup scripts."
|
||||
else
|
||||
echo "There was a problem updating soup scripts. Trying to rerun script update."
|
||||
salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "The soup script has been modified. Please run soup again to continue the upgrade."
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
|
||||
}
|
||||
# Keeping this block in case we need to do a hotfix that requires salt update
|
||||
apply_hotfix() {
|
||||
if [[ "$INSTALLEDVERSION" == "2.4.20" ]] ; then
|
||||
@@ -753,6 +781,9 @@ apply_hotfix() {
|
||||
elastic_fleet_integration_remove endpoints-initial elastic-defend-endpoints
|
||||
/usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
|
||||
elif [[ "$INSTALLEDVERSION" == "2.4.30" ]] ; then
|
||||
if [[ $is_airgap -eq 0 ]]; then
|
||||
update_airgap_rules
|
||||
fi
|
||||
if [[ -f /etc/pki/managerssl.key.old ]]; then
|
||||
echo "Skipping Certificate Generation"
|
||||
else
|
||||
@@ -768,6 +799,7 @@ apply_hotfix() {
|
||||
mv /etc/pki/managerssl.crt /etc/pki/managerssl.crt.old
|
||||
mv /etc/pki/managerssl.key /etc/pki/managerssl.key.old
|
||||
systemctl_func "start" "salt-minion"
|
||||
(wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG"
|
||||
fi
|
||||
else
|
||||
echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)"
|
||||
@@ -875,7 +907,6 @@ main() {
|
||||
echo "Hotfix applied"
|
||||
update_version
|
||||
enable_highstate
|
||||
(wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG"
|
||||
highstate
|
||||
else
|
||||
echo ""
|
||||
@@ -884,9 +915,6 @@ main() {
|
||||
|
||||
systemctl_func "stop" "$cron_service_name"
|
||||
|
||||
# update mine items prior to stopping salt-minion and salt-master
|
||||
update_salt_mine
|
||||
|
||||
echo "Updating dockers to $NEWVERSION."
|
||||
if [[ $is_airgap -eq 0 ]]; then
|
||||
airgap_update_dockers
|
||||
@@ -962,15 +990,8 @@ main() {
|
||||
salt-call state.apply salt.minion -l info queue=True
|
||||
echo ""
|
||||
|
||||
# Only regenerate osquery packages if Fleet is enabled
|
||||
FLEET_MANAGER=$(lookup_pillar fleet_manager)
|
||||
FLEET_NODE=$(lookup_pillar fleet_node)
|
||||
if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then
|
||||
echo ""
|
||||
echo "Regenerating Osquery Packages.... This will take several minutes."
|
||||
salt-call state.apply fleet.event_gen-packages -l info queue=True
|
||||
echo ""
|
||||
fi
|
||||
# ensure the mine is updated and populated before highstates run, following the salt-master restart
|
||||
update_salt_mine
|
||||
|
||||
enable_highstate
|
||||
|
||||
|
||||
@@ -14,11 +14,10 @@ require_manager
|
||||
# Inform user we are about to remove Elastic Fleet data
|
||||
echo
|
||||
echo "This script will remove the current Elastic Fleet install and all of its data and then rerun Elastic Fleet setup."
|
||||
echo "This includes data previously ingested with Fleet such as Zeek and Suricata logs."
|
||||
echo "Deployed Elastic Agents will no longer be enrolled and will need to be reinstalled."
|
||||
echo "This script should only be used as a last resort to reinstall Elastic Fleet."
|
||||
echo
|
||||
echo "If you would like to proceed, type AGREE and hit ENTER."
|
||||
echo "If you would like to proceed, then type AGREE and press ENTER."
|
||||
echo
|
||||
# Read user input
|
||||
read INPUT
|
||||
@@ -33,57 +32,12 @@ so-elastic-fleet-stop --force
|
||||
|
||||
status "Deleting Fleet Data from Pillars..."
|
||||
so-yaml.py remove /opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls elasticfleet
|
||||
sed -i "/fleet_grid_enrollment_token_general.*/d" /opt/so/saltstack/local/pillar/global/soc_global.sls
|
||||
sed -i "/fleet_grid_enrollment_token_heavy.*/d" /opt/so/saltstack/local/pillar/global/soc_global.sls
|
||||
|
||||
status "Deleting Elastic Fleet data..."
|
||||
|
||||
# Check to make sure that Elasticsearch is up & ready
|
||||
RETURN_CODE=0
|
||||
wait_for_web_response "https://localhost:9200/_cat/indices/.kibana*" "green open" 300 "curl -K /opt/so/conf/elasticsearch/curl.config"
|
||||
RETURN_CODE=$?
|
||||
|
||||
if [[ "$RETURN_CODE" != "0" ]]; then
|
||||
status "Elasticsearch not accessible, exiting script..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ALIASES=".fleet-servers .fleet-policies-leader .fleet-agents .fleet-artifacts .fleet-enrollment-api-keys .kibana_ingest"
|
||||
for ALIAS in ${ALIASES}
|
||||
do
|
||||
# Get all concrete indices from alias
|
||||
INDXS=$(curl -K /opt/so/conf/kibana/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/_resolve/index/${ALIAS}" | jq -r '.aliases[].indices[]')
|
||||
|
||||
# Delete all resolved indices
|
||||
for INDX in ${INDXS}
|
||||
do
|
||||
status "Deleting $INDX"
|
||||
curl -K /opt/so/conf/kibana/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/${INDX}" -XDELETE
|
||||
done
|
||||
done
|
||||
|
||||
status "Deleting Fleet-related Data Streams..."
|
||||
DATASTREAMS="logs-suricata-so","logs-kratos-so","logs-soc-so","logs-zeek-so"
|
||||
JSON_STRING=$( jq -n \
|
||||
--arg DATASTREAMLIST "$DATASTREAMS" \
|
||||
'{"dataStreams":[$DATASTREAMLIST]}'
|
||||
)
|
||||
curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/index_management/delete_data_streams" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
|
||||
|
||||
so-yaml.py remove /opt/so/saltstack/local/pillar/global/soc_global.sls global.fleet_grid_enrollment_token_general
|
||||
so-yaml.py remove /opt/so/saltstack/local/pillar/global/soc_global.sls global.fleet_grid_enrollment_token_heavy
|
||||
|
||||
status "Restarting Kibana..."
|
||||
so-kibana-restart --force
|
||||
|
||||
status "Checking to make sure that Kibana API is up & ready..."
|
||||
RETURN_CODE=0
|
||||
wait_for_web_response "http://localhost:5601/api/fleet/settings" "fleet" 300 "curl -K /opt/so/conf/elasticsearch/curl.config"
|
||||
RETURN_CODE=$?
|
||||
|
||||
if [[ "$RETURN_CODE" != "0" ]]; then
|
||||
status "Kibana API not accessible, exiting script..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
status "Removing Integrations State File..."
|
||||
rm -f /opt/so/state/eaintegrations.txt
|
||||
|
||||
@@ -93,4 +47,4 @@ so-elastic-fleet-setup
|
||||
status "Re-installing Elastic Agent on all Grid Nodes..."
|
||||
salt \* state.apply elasticfleet.install_agent_grid queue=True
|
||||
|
||||
status "Elastic Fleet Reset complete...."
|
||||
status "Elastic Fleet Reset complete...."
|
||||
|
||||
0
salt/manager/tools/sbin_jinja/so-yara-update
Executable file → Normal file
0
salt/manager/tools/sbin_jinja/so-yara-update
Executable file → Normal file
Reference in New Issue
Block a user