diff --git a/pyci.sh b/pyci.sh index e85287063..8cbee5e75 100755 --- a/pyci.sh +++ b/pyci.sh @@ -15,12 +15,16 @@ TARGET_DIR=${1:-.} PATH=$PATH:/usr/local/bin -if ! which pytest &> /dev/null || ! which flake8 &> /dev/null ; then - echo "Missing dependencies. Consider running the following command:" - echo " python -m pip install flake8 pytest pytest-cov" +if [ ! -d .venv ]; then + python -m venv .venv +fi + +source .venv/bin/activate + +if ! pip install flake8 pytest pytest-cov pyyaml; then + echo "Unable to install dependencies." exit 1 fi -pip install pytest pytest-cov flake8 "$TARGET_DIR" "--config=${HOME_DIR}/pytest.ini" -python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100 "$TARGET_DIR" \ No newline at end of file +python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100 "$TARGET_DIR" diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 9d943ccab..99d8cb38a 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -93,7 +93,6 @@ firewall: udp: [] kafka: tcp: - - 9092 - 9093 udp: [] kibana: @@ -419,7 +418,6 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni - - kafka heavynode: portgroups: - redis @@ -433,7 +431,6 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni - - kafka receiver: portgroups: - yum @@ -616,7 +613,6 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni - - kafka heavynode: portgroups: - redis @@ -629,7 +625,6 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni - - kafka receiver: portgroups: - yum @@ -639,6 +634,7 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni + - kafka analyst: portgroups: - nginx @@ -818,7 +814,6 @@ firewall: - redis - elasticsearch_rest - elasticsearch_node - - kafka heavynode: portgroups: - docker_registry @@ -828,7 +823,6 @@ firewall: - redis - elasticsearch_rest - elasticsearch_node - - kafka receiver: portgroups: - yum @@ -838,6 +832,7 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni + - kafka analyst: portgroups: - nginx @@ -1282,38 +1277,50 @@ firewall: chain: DOCKER-USER: hostgroups: + desktop: + portgroups: + - elastic_agent_data fleet: portgroups: - - beats_5056 + - elastic_agent_data + idh: + portgroups: + - elastic_agent_data sensor: portgroups: - - beats_5044 - - beats_5644 - elastic_agent_data - - kafka searchnode: portgroups: - redis - - beats_5644 + - elastic_agent_data + standalone: + portgroups: + - redis + - elastic_agent_data - kafka manager: portgroups: + - elastic_agent_data - kafka managersearch: portgroups: - redis - - beats_5644 + - elastic_agent_data - kafka self: portgroups: - redis - - beats_5644 + - elastic_agent_data + - kafka beats_endpoint: portgroups: - beats_5044 beats_endpoint_ssl: portgroups: - beats_5644 + elastic_agent_endpoint: + portgroups: + - elastic_agent_data endgame: portgroups: - endgame diff --git a/salt/manager/defaults.yaml b/salt/manager/defaults.yaml index 8bb34690e..708900af6 100644 --- a/salt/manager/defaults.yaml +++ b/salt/manager/defaults.yaml @@ -2,4 +2,6 @@ manager: reposync: enabled: True hour: 3 - minute: 0 \ No newline at end of file + minute: 0 + additionalCA: '' + insecureSkipVerify: False diff --git a/salt/manager/map.jinja b/salt/manager/map.jinja new file mode 100644 index 000000000..1ab9c12c3 --- /dev/null +++ b/salt/manager/map.jinja @@ -0,0 +1,7 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + +{% import_yaml 'manager/defaults.yaml' as MANAGERDEFAULTS %} +{% set MANAGERMERGED = salt['pillar.get']('manager', MANAGERDEFAULTS.manager, merge=True) %} \ No newline at end of file diff --git a/salt/manager/soc_manager.yaml b/salt/manager/soc_manager.yaml index f6461a0c7..f3346269e 100644 --- a/salt/manager/soc_manager.yaml +++ b/salt/manager/soc_manager.yaml @@ -7,7 +7,7 @@ manager: hour: description: The hour of the day in which the repo sync takes place. global: True - helpLink: soup.html + helpLink: soup.html minute: description: The minute within the hour to run the repo sync. global: True @@ -16,11 +16,23 @@ manager: description: Enable elastalert 1=enabled 0=disabled. global: True helpLink: elastalert.html - no_proxy: - description: String of hosts to ignore the proxy settings for. + no_proxy: + description: String of hosts to ignore the proxy settings for. global: True helpLink: proxy.html proxy: description: Proxy server to use for updates. global: True helpLink: proxy.html + additionalCA: + description: Additional CA certificates to trust in PEM format. + global: True + advanced: True + multiline: True + helpLink: proxy.html + insecureSkipVerify: + description: Disable TLS verification for outgoing requests. This will make your installation less secure to MITM attacks. Recommended only for debugging purposes. + advanced: True + forcedType: bool + global: True + helpLink: proxy.html diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 5ef99b7a9..97b4e8eb4 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -112,8 +112,8 @@ function testMinion() { result=$? # If this so-minion script is not running on the given minion ID, run so-test remotely on the sensor as well - local_id=$(lookup_grain id) - if [[ ! "$local_id" =~ "${MINION_ID}_" ]]; then + local_id=$(lookup_grain id) + if [[ ! "$local_id" =~ "${MINION_ID}_" && "$local_id" != "${MINION_ID}" ]]; then salt "$MINION_ID" cmd.run 'so-test' result=$? fi diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 5427a2e48..275032ee0 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -14,19 +14,20 @@ lockFile = "/tmp/so-yaml.lock" def showUsage(args): - print('Usage: {} [ARGS...]'.format(sys.argv[0])) - print(' General commands:') - print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.') - print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.') - print(' remove - Removes a yaml key, if it exists. Requires KEY arg.') - print(' replace - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.') - print(' help - Prints this usage information.') - print('') - print(' Where:') - print(' YAML_FILE - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml') - print(' KEY - YAML key, does not support \' or " characters at this time. Ex: level1.level2') - print(' VALUE - Value to set for a given key') - print(' LISTITEM - Item to append to a given key\'s list value') + print('Usage: {} [ARGS...]'.format(sys.argv[0]), file=sys.stderr) + print(' General commands:', file=sys.stderr) + print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr) + print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr) + print(' get - Displays (to stdout) the value stored in the given key. Requires KEY arg.', file=sys.stderr) + print(' remove - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr) + print(' replace - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.', file=sys.stderr) + print(' help - Prints this usage information.', file=sys.stderr) + print('', file=sys.stderr) + print(' Where:', file=sys.stderr) + print(' YAML_FILE - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml', file=sys.stderr) + print(' KEY - YAML key, does not support \' or " characters at this time. Ex: level1.level2', file=sys.stderr) + print(' VALUE - Value to set for a given key', file=sys.stderr) + print(' LISTITEM - Item to append to a given key\'s list value', file=sys.stderr) sys.exit(1) @@ -38,7 +39,7 @@ def loadYaml(filename): def writeYaml(filename, content): file = open(filename, "w") - return yaml.dump(content, file) + return yaml.safe_dump(content, file) def appendItem(content, key, listItem): @@ -49,15 +50,15 @@ def appendItem(content, key, listItem): try: content[key].append(listItem) except AttributeError: - print("The existing value for the given key is not a list. No action was taken on the file.") + print("The existing value for the given key is not a list. No action was taken on the file.", file=sys.stderr) return 1 except KeyError: - print("The key provided does not exist. No action was taken on the file.") + print("The key provided does not exist. No action was taken on the file.", file=sys.stderr) return 1 def convertType(value): - if len(value) > 0 and (not value.startswith("0") or len(value) == 1): + if isinstance(value, str) and len(value) > 0 and (not value.startswith("0") or len(value) == 1): if "." in value: try: value = float(value) @@ -83,7 +84,7 @@ def append(args): if len(args) != 3: print('Missing filename, key arg, or list item to append', file=sys.stderr) showUsage(None) - return + return 1 filename = args[0] key = args[1] @@ -112,7 +113,7 @@ def add(args): if len(args) != 3: print('Missing filename, key arg, and/or value', file=sys.stderr) showUsage(None) - return + return 1 filename = args[0] key = args[1] @@ -137,7 +138,7 @@ def remove(args): if len(args) != 2: print('Missing filename or key arg', file=sys.stderr) showUsage(None) - return + return 1 filename = args[0] key = args[1] @@ -153,7 +154,7 @@ def replace(args): if len(args) != 3: print('Missing filename, key arg, and/or value', file=sys.stderr) showUsage(None) - return + return 1 filename = args[0] key = args[1] @@ -167,6 +168,32 @@ def replace(args): return 0 +def getKeyValue(content, key): + pieces = key.split(".", 1) + if len(pieces) > 1 and pieces[0] in content: + return getKeyValue(content[pieces[0]], pieces[1]) + return content.get(key, None) + + +def get(args): + if len(args) != 2: + print('Missing filename or key arg', file=sys.stderr) + showUsage(None) + return 1 + + filename = args[0] + key = args[1] + + content = loadYaml(filename) + output = getKeyValue(content, key) + if output is None: + print("Not found", file=sys.stderr) + return 2 + + print(yaml.safe_dump(output)) + return 0 + + def main(): args = sys.argv[1:] @@ -178,6 +205,7 @@ def main(): "help": showUsage, "add": add, "append": append, + "get": get, "remove": remove, "replace": replace, } @@ -195,11 +223,11 @@ def main(): break except Exception: if lockAttempts == 1: - print("Waiting for lock file to be released from another process...") + print("Waiting for lock file to be released from another process...", file=sys.stderr) time.sleep(2) if lockAttempts == maxAttempts: - print("Lock file (" + lockFile + ") could not be created; proceeding without lock.") + print("Lock file (" + lockFile + ") could not be created; proceeding without lock.", file=sys.stderr) cmd = commands.get(args[0], showUsage) code = cmd(args[1:]) diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index 7effabac9..5ca46cb68 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -15,40 +15,40 @@ class TestRemove(unittest.TestCase): def test_main_missing_input(self): with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd"] soyaml.main() sysmock.assert_called_once_with(1) - self.assertIn(mock_stdout.getvalue(), "Usage:") + self.assertIn("Usage:", mock_stderr.getvalue()) def test_main_help_locked(self): filename = "/tmp/so-yaml.lock" file = open(filename, "w") file.write = "fake lock file" with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: with patch('time.sleep', new=MagicMock()) as mock_sleep: sys.argv = ["cmd", "help"] soyaml.main() sysmock.assert_called() mock_sleep.assert_called_with(2) - self.assertIn(mock_stdout.getvalue(), "Usage:") + self.assertIn("Usage:", mock_stderr.getvalue()) def test_main_help(self): with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "help"] soyaml.main() sysmock.assert_called() - self.assertIn(mock_stdout.getvalue(), "Usage:") + self.assertIn("Usage:", mock_stderr.getvalue()) def test_remove_missing_arg(self): with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "help"] soyaml.remove(["file"]) sysmock.assert_called() - self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n") + self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue()) def test_remove(self): filename = "/tmp/so-yaml_test-remove.yaml" @@ -97,7 +97,7 @@ class TestRemove(unittest.TestCase): def test_remove_missing_args(self): with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: filename = "/tmp/so-yaml_test-remove.yaml" file = open(filename, "w") file.write("{key1: { child1: 123, child2: abc }, key2: false}") @@ -112,15 +112,15 @@ class TestRemove(unittest.TestCase): expected = "{key1: { child1: 123, child2: abc }, key2: false}" self.assertEqual(actual, expected) sysmock.assert_called_once_with(1) - self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n") + self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue()) def test_append_missing_arg(self): with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "help"] soyaml.append(["file", "key"]) sysmock.assert_called() - self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, or list item to append\n") + self.assertIn("Missing filename, key arg, or list item to append\n", mock_stderr.getvalue()) def test_append(self): filename = "/tmp/so-yaml_test-remove.yaml" @@ -173,11 +173,11 @@ class TestRemove(unittest.TestCase): file.close() with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stdout', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "append", filename, "key4", "h"] soyaml.main() sysmock.assert_called() - self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n") + self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue()) def test_append_key_noexist_deep(self): filename = "/tmp/so-yaml_test-append.yaml" @@ -186,11 +186,11 @@ class TestRemove(unittest.TestCase): file.close() with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stdout', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "append", filename, "key1.child2.deep3", "h"] soyaml.main() sysmock.assert_called() - self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n") + self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue()) def test_append_key_nonlist(self): filename = "/tmp/so-yaml_test-append.yaml" @@ -199,11 +199,11 @@ class TestRemove(unittest.TestCase): file.close() with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stdout', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "append", filename, "key1", "h"] soyaml.main() sysmock.assert_called() - self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n") + self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue()) def test_append_key_nonlist_deep(self): filename = "/tmp/so-yaml_test-append.yaml" @@ -212,11 +212,11 @@ class TestRemove(unittest.TestCase): file.close() with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stdout', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"] soyaml.main() sysmock.assert_called() - self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n") + self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue()) def test_add_key(self): content = {} @@ -244,11 +244,11 @@ class TestRemove(unittest.TestCase): def test_add_missing_arg(self): with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "help"] soyaml.add(["file", "key"]) sysmock.assert_called() - self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n") + self.assertIn("Missing filename, key arg, and/or value\n", mock_stderr.getvalue()) def test_add(self): filename = "/tmp/so-yaml_test-add.yaml" @@ -296,11 +296,11 @@ class TestRemove(unittest.TestCase): def test_replace_missing_arg(self): with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "help"] soyaml.replace(["file", "key"]) sysmock.assert_called() - self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n") + self.assertIn("Missing filename, key arg, and/or value\n", mock_stderr.getvalue()) def test_replace(self): filename = "/tmp/so-yaml_test-add.yaml" @@ -360,3 +360,77 @@ class TestRemove(unittest.TestCase): self.assertEqual(soyaml.convertType("false"), False) self.assertEqual(soyaml.convertType("FALSE"), False) self.assertEqual(soyaml.convertType(""), "") + + def test_get_int(self): + with patch('sys.stdout', new=StringIO()) as mock_stdout: + filename = "/tmp/so-yaml_test-get.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}") + file.close() + + result = soyaml.get([filename, "key1.child2.deep1"]) + self.assertEqual(result, 0) + self.assertIn("45\n...", mock_stdout.getvalue()) + + def test_get_str(self): + with patch('sys.stdout', new=StringIO()) as mock_stdout: + filename = "/tmp/so-yaml_test-get.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}") + file.close() + + result = soyaml.get([filename, "key1.child2.deep1"]) + self.assertEqual(result, 0) + self.assertIn("hello\n...", mock_stdout.getvalue()) + + def test_get_list(self): + with patch('sys.stdout', new=StringIO()) as mock_stdout: + filename = "/tmp/so-yaml_test-get.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}") + file.close() + + result = soyaml.get([filename, "key3"]) + self.assertEqual(result, 0) + self.assertIn("- e\n- f\n- g\n", mock_stdout.getvalue()) + + def test_get_dict(self): + with patch('sys.stdout', new=StringIO()) as mock_stdout: + filename = "/tmp/so-yaml_test-get.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}") + file.close() + + result = soyaml.get([filename, "key1"]) + self.assertEqual(result, 0) + self.assertIn("child1: 123\nchild2:\n deep1: hello\n", mock_stdout.getvalue()) + + def test_get_missing(self): + with patch('sys.stdout', new=StringIO()) as mock_stdout: + filename = "/tmp/so-yaml_test-get.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}") + file.close() + + result = soyaml.get([filename, "key1.child2.deep3"]) + self.assertEqual(result, 2) + self.assertEqual("", mock_stdout.getvalue()) + + def test_get_missing_parent(self): + with patch('sys.stdout', new=StringIO()) as mock_stdout: + filename = "/tmp/so-yaml_test-get.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}") + file.close() + + result = soyaml.get([filename, "key1.child3.deep3"]) + self.assertEqual(result, 2) + self.assertEqual("", mock_stdout.getvalue()) + + def test_get_usage(self): + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stderr: + result = soyaml.get([]) + self.assertEqual(result, 1) + self.assertIn("Missing filename or key arg", mock_stderr.getvalue()) + sysmock.assert_called_once_with(1) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 95ca60b95..56d2d7de3 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -19,6 +19,9 @@ SOUP_LOG=/root/soup.log WHATWOULDYOUSAYYAHDOHERE=soup whiptail_title='Security Onion UPdater' NOTIFYCUSTOMELASTICCONFIG=false +# used to display messages to the user at the end of soup +declare -a FINAL_MESSAGE_QUEUE=() + check_err() { local exit_code=$1 @@ -344,6 +347,29 @@ masterunlock() { mv -v $BACKUPTOPFILE $TOPFILE } +phases_pillar_2_4_80() { + echo "Checking if pillar value: elasticsearch.index_settings.global_overrides.index_template.phases exists" + set +e + PHASES=$(so-yaml.py get /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases) + case $? in + 0) + so-yaml.py remove /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases + read -r -d '' msg <<- EOF +Found elasticsearch.index_settings.global_overrides.index_template.phases was set to: +${PHASES} + +Removed unused pillar value: elasticsearch.index_settings.global_overrides.index_template.phases +To set policies, navigate to the SOC Grid Configuration UI at elasticsearch.index_settings.global_overrides.policy.phases +A backup of all pillar files was saved to /nsm/backup/ +EOF + FINAL_MESSAGE_QUEUE+=("$msg") + ;; + 2) echo "Pillar elasticsearch.index_settings.global_overrides.index_template.phases does not exist. No action taken." ;; + *) echo "so-yaml.py returned something other than 0 or 2 exit code" ;; # we shouldn't see this + esac + set -e +} + preupgrade_changes() { # This function is to add any new pillar items if needed. echo "Checking to see if changes are needed." @@ -604,6 +630,7 @@ up_to_2.4.70() { } up_to_2.4.80() { + phases_pillar_2_4_80 # Kafka configuration changes # Global pipeline changes to REDIS or KAFKA @@ -1282,6 +1309,15 @@ EOF fi +# check if the FINAL_MESSAGE_QUEUE is not empty +if (( ${#FINAL_MESSAGE_QUEUE[@]} != 0 )); then + echo "The following additional information applies specifically to your grid:\n" + for m in "${FINAL_MESSAGE_QUEUE[@]}"; do + echo "$m" + echo + done +fi + echo "### soup has been served at $(date) ###" } diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index f2c88fde9..c823175cb 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -1,16 +1,21 @@ {# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one - or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at https://securityonion.net/license; you may not use this file except in compliance with the Elastic License 2.0. #} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %} {% from 'logstash/map.jinja' import LOGSTASH_NODES %} +{% from 'manager/map.jinja' import MANAGERMERGED %} {% set DOCKER_EXTRA_HOSTS = LOGSTASH_NODES %} {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %} {% set SOCMERGED = salt['pillar.get']('soc', SOCDEFAULTS, merge=true) %} +{% do SOCMERGED.config.server.update({'proxy': MANAGERMERGED.proxy}) %} +{% do SOCMERGED.config.server.update({'additionalCA': MANAGERMERGED.additionalCA}) %} +{% do SOCMERGED.config.server.update({'insecureSkipVerify': MANAGERMERGED.insecureSkipVerify}) %} + {# if SOCMERGED.config.server.modules.cases == httpcase details come from the soc pillar #} {% if SOCMERGED.config.server.modules.cases != 'soc' %} {% do SOCMERGED.config.server.modules.elastic.update({'casesEnabled': false}) %} diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index e157ff852..1ecabacd8 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -150,13 +150,16 @@ suricata: helpLink: suricata.html vars: address-groups: - HOME_NET: &suriaddressgroup + HOME_NET: description: Assign a list of hosts, or networks, using CIDR notation, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable. regex: ^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\/([0-9]|[1-2][0-9]|3[0-2]))?$|^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?))|:))|(([0-9A-Fa-f]{1,4}:){5}((:[0-9A-Fa-f]{1,4}){1,2}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){4}((:[0-9A-Fa-f]{1,4}){1,3}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){3}((:[0-9A-Fa-f]{1,4}){1,4}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){2}((:[0-9A-Fa-f]{1,4}){1,5}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){1}((:[0-9A-Fa-f]{1,4}){1,6}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(:((:[0-9A-Fa-f]{1,4}){1,7}|:)))(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$ regexFailureMessage: You must enter a valid IP address or CIDR. helpLink: suricata.html duplicates: True - EXTERNAL_NET: *suriaddressgroup + EXTERNAL_NET: &suriaddressgroup + description: Assign a list of hosts, or networks, or other customization, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable. + helpLink: suricata.html + duplicates: True HTTP_SERVERS: *suriaddressgroup SMTP_SERVERS: *suriaddressgroup SQL_SERVERS: *suriaddressgroup