diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index cef47168f..ec8c8337e 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -32,6 +32,16 @@ so-elastic-fleet-auto-configure-logstash-outputs: - retry: attempts: 4 interval: 30 + +{# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #} +so-elastic-fleet-auto-configure-logstash-outputs-force: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-outputs-update --force --certs + - retry: + attempts: 4 + interval: 30 + - onchanges: + - x509: etc_elasticfleet_logstash_crt {% endif %} # If enabled, automatically update Fleet Server URLs & ES Connection diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update index 9efe8a19d..4fa68298c 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -8,6 +8,27 @@ . /usr/sbin/so-common +FORCE_UPDATE=false +UPDATE_CERTS=false + +while [[ $# -gt 0 ]]; do + case $1 in + -f|--force) + FORCE_UPDATE=true + shift + ;; + -c| --certs) + UPDATE_CERTS=true + shift + ;; + *) + echo "Unknown option $1" + echo "Usage: $0 [-f|--force] [-c|--certs]" + exit 1 + ;; + esac +done + # Only run on Managers if ! is_manager_node; then printf "Not a Manager Node... Exiting" @@ -17,17 +38,42 @@ fi function update_logstash_outputs() { if logstash_policy=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_logstash" --retry 3 --retry-delay 10 --fail 2>/dev/null); then SSL_CONFIG=$(echo "$logstash_policy" | jq -r '.item.ssl') + LOGSTASHKEY=$(openssl rsa -in /etc/pki/elasticfleet-logstash.key) + LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt) + LOGSTASHCA=$(openssl x509 -in /etc/pki/tls/certs/intca.crt) if SECRETS=$(echo "$logstash_policy" | jq -er '.item.secrets' 2>/dev/null); then - JSON_STRING=$(jq -n \ - --arg UPDATEDLIST "$NEW_LIST_JSON" \ - --argjson SECRETS "$SECRETS" \ - --argjson SSL_CONFIG "$SSL_CONFIG" \ - '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG,"secrets": $SECRETS}') + if [[ "$UPDATE_CERTS" != "true" ]]; then + # Reuse existing secret + JSON_STRING=$(jq -n \ + --arg UPDATEDLIST "$NEW_LIST_JSON" \ + --argjson SECRETS "$SECRETS" \ + --argjson SSL_CONFIG "$SSL_CONFIG" \ + '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG,"secrets": $SECRETS}') + else + # Update certs, creating new secret + JSON_STRING=$(jq -n \ + --arg UPDATEDLIST "$NEW_LIST_JSON" \ + --arg LOGSTASHKEY "$LOGSTASHKEY" \ + --arg LOGSTASHCRT "$LOGSTASHCRT" \ + --arg LOGSTASHCA "$LOGSTASHCA" \ + '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": {"certificate": $LOGSTASHCRT,"certificate_authorities":[ $LOGSTASHCA ]},"secrets": {"ssl":{"key": $LOGSTASHKEY }}}') + fi else - JSON_STRING=$(jq -n \ - --arg UPDATEDLIST "$NEW_LIST_JSON" \ - --argjson SSL_CONFIG "$SSL_CONFIG" \ - '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG}') + if [[ "$UPDATE_CERTS" != "true" ]]; then + # Reuse existing ssl config + JSON_STRING=$(jq -n \ + --arg UPDATEDLIST "$NEW_LIST_JSON" \ + --argjson SSL_CONFIG "$SSL_CONFIG" \ + '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG}') + else + # Update ssl config + JSON_STRING=$(jq -n \ + --arg UPDATEDLIST "$NEW_LIST_JSON" \ + --arg LOGSTASHKEY "$LOGSTASHKEY" \ + --arg LOGSTASHCRT "$LOGSTASHCRT" \ + --arg LOGSTASHCA "$LOGSTASHCA" \ + '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": {"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}') + fi fi fi @@ -151,7 +197,7 @@ NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "$ NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') # Compare the current & new list of outputs - if different, update the Logstash outputs -if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then +if [[ "$NEW_HASH" = "$CURRENT_HASH" ]] && [[ "$FORCE_UPDATE" != "true" ]]; then printf "\nHashes match - no update needed.\n" printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 4c8544893..00290f18b 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -17,6 +17,7 @@ def showUsage(args): print('Usage: {} [ARGS...]'.format(sys.argv[0]), file=sys.stderr) print(' General commands:', file=sys.stderr) print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr) + print(' removelistitem - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr) print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr) print(' get - Displays (to stdout) the value stored in the given key. Requires KEY arg.', file=sys.stderr) print(' remove - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr) @@ -57,6 +58,24 @@ def appendItem(content, key, listItem): return 1 +def removeListItem(content, key, listItem): + pieces = key.split(".", 1) + if len(pieces) > 1: + removeListItem(content[pieces[0]], pieces[1], listItem) + else: + try: + if not isinstance(content[key], list): + raise AttributeError("Value is not a list") + if listItem in content[key]: + content[key].remove(listItem) + except (AttributeError, TypeError): + print("The existing value for the given key is not a list. No action was taken on the file.", file=sys.stderr) + return 1 + except KeyError: + print("The key provided does not exist. No action was taken on the file.", file=sys.stderr) + return 1 + + def convertType(value): if isinstance(value, str) and value.startswith("file:"): path = value[5:] # Remove "file:" prefix @@ -103,6 +122,23 @@ def append(args): return 0 +def removelistitem(args): + if len(args) != 3: + print('Missing filename, key arg, or list item to remove', file=sys.stderr) + showUsage(None) + return 1 + + filename = args[0] + key = args[1] + listItem = args[2] + + content = loadYaml(filename) + removeListItem(content, key, convertType(listItem)) + writeYaml(filename, content) + + return 0 + + def addKey(content, key, value): pieces = key.split(".", 1) if len(pieces) > 1: @@ -211,6 +247,7 @@ def main(): "help": showUsage, "add": add, "append": append, + "removelistitem": removelistitem, "get": get, "remove": remove, "replace": replace, diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index f33c1300a..3b5ec498e 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -457,3 +457,126 @@ class TestRemove(unittest.TestCase): self.assertEqual(result, 1) self.assertIn("Missing filename or key arg", mock_stderr.getvalue()) sysmock.assert_called_once_with(1) + + +class TestRemoveListItem(unittest.TestCase): + + def test_removelistitem_missing_arg(self): + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stderr: + sys.argv = ["cmd", "help"] + soyaml.removelistitem(["file", "key"]) + sysmock.assert_called() + self.assertIn("Missing filename, key arg, or list item to remove", mock_stderr.getvalue()) + + def test_removelistitem(self): + filename = "/tmp/so-yaml_test-removelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}") + file.close() + + soyaml.removelistitem([filename, "key3", "b"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2: abc\nkey2: false\nkey3:\n- a\n- c\n" + self.assertEqual(actual, expected) + + def test_removelistitem_nested(self): + filename = "/tmp/so-yaml_test-removelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}") + file.close() + + soyaml.removelistitem([filename, "key1.child2", "b"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2:\n - a\n - c\nkey2: false\nkey3:\n- e\n- f\n- g\n" + self.assertEqual(actual, expected) + + def test_removelistitem_nested_deep(self): + filename = "/tmp/so-yaml_test-removelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + soyaml.removelistitem([filename, "key1.child2.deep2", "b"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2:\n deep1: 45\n deep2:\n - a\n - c\nkey2: false\nkey3:\n- e\n- f\n- g\n" + self.assertEqual(actual, expected) + + def test_removelistitem_item_not_in_list(self): + filename = "/tmp/so-yaml_test-removelistitem.yaml" + file = open(filename, "w") + file.write("{key1: [a,b,c]}") + file.close() + + soyaml.removelistitem([filename, "key1", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- a\n- b\n- c\n" + self.assertEqual(actual, expected) + + def test_removelistitem_key_noexist(self): + filename = "/tmp/so-yaml_test-removelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stderr: + sys.argv = ["cmd", "removelistitem", filename, "key4", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue()) + + def test_removelistitem_key_noexist_deep(self): + filename = "/tmp/so-yaml_test-removelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stderr: + sys.argv = ["cmd", "removelistitem", filename, "key1.child2.deep3", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue()) + + def test_removelistitem_key_nonlist(self): + filename = "/tmp/so-yaml_test-removelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stderr: + sys.argv = ["cmd", "removelistitem", filename, "key1", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue()) + + def test_removelistitem_key_nonlist_deep(self): + filename = "/tmp/so-yaml_test-removelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stderr: + sys.argv = ["cmd", "removelistitem", filename, "key1.child2.deep1", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue()) diff --git a/setup/so-functions b/setup/so-functions index a8414d0e8..3ac27f26b 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1604,16 +1604,21 @@ proxy_validate() { reserve_group_ids() { # This is a hack to fix OS from taking group IDs that we need + logCmd "groupadd -g 920 docker" logCmd "groupadd -g 928 kratos" logCmd "groupadd -g 930 elasticsearch" logCmd "groupadd -g 931 logstash" logCmd "groupadd -g 932 kibana" logCmd "groupadd -g 933 elastalert" logCmd "groupadd -g 937 zeek" + logCmd "groupadd -g 938 salt" + logCmd "groupadd -g 939 socore" logCmd "groupadd -g 940 suricata" + logCmd "groupadd -g 948 elastic-agent-pr" + logCmd "groupadd -g 949 elastic-agent" logCmd "groupadd -g 941 stenographer" - logCmd "groupadd -g 945 ossec" - logCmd "groupadd -g 946 cyberchef" + logCmd "groupadd -g 947 elastic-fleet" + logCmd "groupadd -g 960 kafka" } reserve_ports() { diff --git a/setup/so-setup b/setup/so-setup index 91f1fa9aa..d09e8fc35 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -682,6 +682,8 @@ if ! [[ -f $install_opt_file ]]; then fi info "Reserving ports" reserve_ports + info "Reserving group ids" + reserve_group_ids info "Setting Paths" # Set the paths set_path @@ -840,7 +842,10 @@ if ! [[ -f $install_opt_file ]]; then if [[ $monints ]]; then configure_network_sensor fi + info "Reserving ports" reserve_ports + info "Reserving group ids" + reserve_group_ids # Set the version mark_version # Disable the setup from prompting at login