From 3aea2dec85745aca62f89a53cb6f8444629dce32 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 1 Apr 2024 09:50:18 -0400 Subject: [PATCH] analytics --- salt/manager/tools/sbin/so-yaml.py | 82 +++++++++++- salt/manager/tools/sbin/so-yaml_test.py | 159 ++++++++++++++++++++++++ salt/manager/tools/sbin/soup | 42 +++++++ salt/soc/config.sls | 9 ++ salt/soc/defaults.yaml | 1 + salt/soc/enabled.sls | 5 + salt/soc/files/soc/analytics.js | 5 + salt/soc/files/soc/motd.md | 4 + salt/soc/soc_soc.yaml | 5 + setup/so-functions | 4 + setup/so-setup | 5 + setup/so-whiptail | 20 +++ 12 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 salt/soc/files/soc/analytics.js diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 41cab0b23..5427a2e48 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -17,13 +17,16 @@ def showUsage(args): print('Usage: {} [ARGS...]'.format(sys.argv[0])) print(' General commands:') print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.') + print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.') print(' remove - Removes a yaml key, if it exists. Requires KEY arg.') + print(' replace - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.') print(' help - Prints this usage information.') print('') print(' Where:') print(' YAML_FILE - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml') print(' KEY - YAML key, does not support \' or " characters at this time. Ex: level1.level2') - print(' LISTITEM - Item to add to the list.') + print(' VALUE - Value to set for a given key') + print(' LISTITEM - Item to append to a given key\'s list value') sys.exit(1) @@ -37,6 +40,7 @@ def writeYaml(filename, content): file = open(filename, "w") return yaml.dump(content, file) + def appendItem(content, key, listItem): pieces = key.split(".", 1) if len(pieces) > 1: @@ -51,6 +55,30 @@ def appendItem(content, key, listItem): print("The key provided does not exist. No action was taken on the file.") return 1 + +def convertType(value): + if len(value) > 0 and (not value.startswith("0") or len(value) == 1): + if "." in value: + try: + value = float(value) + return value + except ValueError: + pass + + try: + value = int(value) + return value + except ValueError: + pass + + lowered_value = value.lower() + if lowered_value == "false": + return False + elif lowered_value == "true": + return True + return value + + def append(args): if len(args) != 3: print('Missing filename, key arg, or list item to append', file=sys.stderr) @@ -62,11 +90,41 @@ def append(args): listItem = args[2] content = loadYaml(filename) - appendItem(content, key, listItem) + appendItem(content, key, convertType(listItem)) writeYaml(filename, content) return 0 + +def addKey(content, key, value): + pieces = key.split(".", 1) + if len(pieces) > 1: + if not pieces[0] in content: + content[pieces[0]] = {} + addKey(content[pieces[0]], pieces[1], value) + elif key in content: + raise KeyError("key already exists") + else: + content[key] = value + + +def add(args): + if len(args) != 3: + print('Missing filename, key arg, and/or value', file=sys.stderr) + showUsage(None) + return + + filename = args[0] + key = args[1] + value = args[2] + + content = loadYaml(filename) + addKey(content, key, convertType(value)) + writeYaml(filename, content) + + return 0 + + def removeKey(content, key): pieces = key.split(".", 1) if len(pieces) > 1: @@ -91,6 +149,24 @@ def remove(args): return 0 +def replace(args): + if len(args) != 3: + print('Missing filename, key arg, and/or value', file=sys.stderr) + showUsage(None) + return + + filename = args[0] + key = args[1] + value = args[2] + + content = loadYaml(filename) + removeKey(content, key) + addKey(content, key, convertType(value)) + writeYaml(filename, content) + + return 0 + + def main(): args = sys.argv[1:] @@ -100,8 +176,10 @@ def main(): commands = { "help": showUsage, + "add": add, "append": append, "remove": remove, + "replace": replace, } code = 1 diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index 488877ea1..7effabac9 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -42,6 +42,14 @@ class TestRemove(unittest.TestCase): sysmock.assert_called() self.assertIn(mock_stdout.getvalue(), "Usage:") + def test_remove_missing_arg(self): + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "help"] + soyaml.remove(["file"]) + sysmock.assert_called() + self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n") + def test_remove(self): filename = "/tmp/so-yaml_test-remove.yaml" file = open(filename, "w") @@ -106,6 +114,14 @@ class TestRemove(unittest.TestCase): sysmock.assert_called_once_with(1) self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n") + def test_append_missing_arg(self): + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "help"] + soyaml.append(["file", "key"]) + sysmock.assert_called() + self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, or list item to append\n") + def test_append(self): filename = "/tmp/so-yaml_test-remove.yaml" file = open(filename, "w") @@ -201,3 +217,146 @@ class TestRemove(unittest.TestCase): soyaml.main() sysmock.assert_called() self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n") + + def test_add_key(self): + content = {} + soyaml.addKey(content, "foo", 123) + self.assertEqual(content, {"foo": 123}) + + try: + soyaml.addKey(content, "foo", "bar") + self.assertFail("expected key error since key already exists") + except KeyError: + pass + + try: + soyaml.addKey(content, "foo.bar", 123) + self.assertFail("expected type error since key parent value is not a map") + except TypeError: + pass + + content = {} + soyaml.addKey(content, "foo", "bar") + self.assertEqual(content, {"foo": "bar"}) + + soyaml.addKey(content, "badda.badda", "boom") + self.assertEqual(content, {"foo": "bar", "badda": {"badda": "boom"}}) + + def test_add_missing_arg(self): + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "help"] + soyaml.add(["file", "key"]) + sysmock.assert_called() + self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n") + + def test_add(self): + filename = "/tmp/so-yaml_test-add.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}") + file.close() + + soyaml.add([filename, "key4", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + expected = "key1:\n child1: 123\n child2: abc\nkey2: false\nkey3:\n- a\n- b\n- c\nkey4: d\n" + self.assertEqual(actual, expected) + + def test_add_nested(self): + filename = "/tmp/so-yaml_test-add.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}") + file.close() + + soyaml.add([filename, "key1.child3", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2:\n - a\n - b\n - c\n child3: d\nkey2: false\nkey3:\n- e\n- f\n- g\n" + self.assertEqual(actual, expected) + + def test_add_nested_deep(self): + filename = "/tmp/so-yaml_test-add.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}") + file.close() + + soyaml.add([filename, "key1.child2.deep2", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2:\n deep1: 45\n deep2: d\nkey2: false\nkey3:\n- e\n- f\n- g\n" + self.assertEqual(actual, expected) + + def test_replace_missing_arg(self): + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "help"] + soyaml.replace(["file", "key"]) + sysmock.assert_called() + self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n") + + def test_replace(self): + filename = "/tmp/so-yaml_test-add.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}") + file.close() + + soyaml.replace([filename, "key2", True]) + + file = open(filename, "r") + actual = file.read() + file.close() + expected = "key1:\n child1: 123\n child2: abc\nkey2: true\nkey3:\n- a\n- b\n- c\n" + self.assertEqual(actual, expected) + + def test_replace_nested(self): + filename = "/tmp/so-yaml_test-add.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}") + file.close() + + soyaml.replace([filename, "key1.child2", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2: d\nkey2: false\nkey3:\n- e\n- f\n- g\n" + self.assertEqual(actual, expected) + + def test_replace_nested_deep(self): + filename = "/tmp/so-yaml_test-add.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}") + file.close() + + soyaml.replace([filename, "key1.child2.deep1", 46]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2:\n deep1: 46\nkey2: false\nkey3:\n- e\n- f\n- g\n" + self.assertEqual(actual, expected) + + def test_convert(self): + self.assertEqual(soyaml.convertType("foo"), "foo") + self.assertEqual(soyaml.convertType("foo.bar"), "foo.bar") + self.assertEqual(soyaml.convertType("123"), 123) + self.assertEqual(soyaml.convertType("0"), 0) + self.assertEqual(soyaml.convertType("00"), "00") + self.assertEqual(soyaml.convertType("0123"), "0123") + self.assertEqual(soyaml.convertType("123.456"), 123.456) + self.assertEqual(soyaml.convertType("0123.456"), "0123.456") + self.assertEqual(soyaml.convertType("true"), True) + self.assertEqual(soyaml.convertType("TRUE"), True) + self.assertEqual(soyaml.convertType("false"), False) + self.assertEqual(soyaml.convertType("FALSE"), False) + self.assertEqual(soyaml.convertType(""), "") diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index a585f877c..db5335a7a 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -357,6 +357,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.30 ]] && up_to_2.4.40 [[ "$INSTALLEDVERSION" == 2.4.40 ]] && up_to_2.4.50 [[ "$INSTALLEDVERSION" == 2.4.50 ]] && up_to_2.4.60 + [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70 true } @@ -373,6 +374,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.30 ]] && post_to_2.4.40 [[ "$POSTVERSION" == 2.4.40 ]] && post_to_2.4.50 [[ "$POSTVERSION" == 2.4.50 ]] && post_to_2.4.60 + [[ "$POSTVERSION" == 2.4.60 ]] && post_to_2.4.70 true } @@ -435,6 +437,11 @@ post_to_2.4.60() { POSTVERSION=2.4.60 } +post_to_2.4.70() { + echo "Nothing to apply" + POSTVERSION=2.4.70 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -574,6 +581,41 @@ up_to_2.4.60() { INSTALLEDVERSION=2.4.60 } +up_to_2.4.70() { + if [[ -z $UNATTENDED && $is_airgap -ne 0 ]]; then + cat << ASSIST_EOF + +--------------- SOC Telemetry --------------- + +The Security Onion development team could use your help! Enabling SOC +Telemetry will help the team understand which UI features are being +used and enables informed prioritization of future development. + +Adjust this setting at anytime via the SOC Configuration screen. + +For more information visit https://docs.securityonion.net/telemetry.rst. + +ASSIST_EOF + + echo -n "Continue the upgrade with SOC Telemetry enabled [Y/n]? " + + read -r input + input=$(echo "${input,,}" | xargs echo -n) + echo "" + if [[ ${#input} -eq 0 || "$input" == "yes" || "$input" == "y" || "$input" == "yy" ]]; then + echo "Thank you for helping improve Security Onion!" + else + if so-yaml.py replace /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.telemetryEnabled false; then + echo "Disabled SOC Telemetry." + else + fail "Failed to disable SOC Telemetry; aborting." + fi + fi + echo "" + fi + INSTALLEDVERSION=2.4.70 +} + determine_elastic_agent_upgrade() { if [[ $is_airgap -eq 0 ]]; then update_elastic_agent_airgap diff --git a/salt/soc/config.sls b/salt/soc/config.sls index ad0ab1c8d..3e756f977 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -52,6 +52,15 @@ socsaltdir: - mode: 770 - makedirs: True +socanalytics: + file.managed: + - name: /opt/so/conf/soc/analytics.js + - source: salt://soc/files/soc/analytics.js + - user: 939 + - group: 939 + - mode: 600 + - show_changes: False + socconfig: file.managed: - name: /opt/so/conf/soc/soc.json diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 861f6b02c..2ba99cd11 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1,5 +1,6 @@ soc: enabled: False + telemetryEnabled: true config: logFilename: /opt/sensoroni/logs/sensoroni-server.log logLevel: info diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index bbe36e5b7..6cea0c70d 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -8,6 +8,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} {% from 'soc/merged.map.jinja' import DOCKER_EXTRA_HOSTS %} +{% from 'soc/merged.map.jinja' import SOCMERGED %} include: - soc.config @@ -31,6 +32,9 @@ so-soc: - /nsm/soc/uploads:/nsm/soc/uploads:rw - /opt/so/log/soc/:/opt/sensoroni/logs/:rw - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro +{% if SOCMERGED.telemetryEnabled and not GLOBALS.airgap %} + - /opt/so/conf/soc/analytics.js:/opt/sensoroni/html/js/analytics.js:ro +{% endif %} - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro - /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro @@ -67,6 +71,7 @@ so-soc: - file: socdatadir - file: soclogdir - file: socconfig + - file: socanalytics - file: socmotd - file: socbanner - file: soccustom diff --git a/salt/soc/files/soc/analytics.js b/salt/soc/files/soc/analytics.js new file mode 100644 index 000000000..6a0d72d5d --- /dev/null +++ b/salt/soc/files/soc/analytics.js @@ -0,0 +1,5 @@ +(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': + new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], + j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= + 'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); + })(window,document,'script','dataLayer','GTM-TM46SL7T'); diff --git a/salt/soc/files/soc/motd.md b/salt/soc/files/soc/motd.md index d6b0d3d27..005a2be0f 100644 --- a/salt/soc/files/soc/motd.md +++ b/salt/soc/files/soc/motd.md @@ -12,6 +12,10 @@ To see all the latest features and fixes in this version of Security Onion, clic Want the best hardware for your enterprise deployment? Check out our [enterprise appliances](https://securityonionsolutions.com/hardware/)! +## Premium Support + +Experiencing difficulties and need priority support or remote assistance? We offer a [premium support plan](https://securityonionsolutions.com/support/) to assist corporate, educational, and government organizations. + ## Customize This Space Make this area your own by customizing the content in the [Config](/#/config?s=soc.files.soc.motd__md) interface. diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index eae52e31b..eed0113fc 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -2,6 +2,11 @@ soc: enabled: description: You can enable or disable SOC. advanced: True + telemetryEnabled: + title: SOC Telemetry + description: When enabled, SOC provides feature usage data to the Security Onion development team via Google Analytics. This data helps Security Onion developers determine which product features are being used and can also provide insight into improving the user interface. When changing this setting, wait for the grid to fully synchronize and then perform a hard browser refresh on SOC, to force the browser cache to update and reflect the new setting. + global: True + helpLink: telemetry.html files: soc: banner__md: diff --git a/setup/so-functions b/setup/so-functions index 0d66a2621..3a0da7bda 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1258,6 +1258,10 @@ soc_pillar() { " server:"\ " srvKey: '$SOCSRVKEY'"\ "" > "$soc_pillar_file" + + if [[ $telemetry -ne 0 ]]; then + echo " telemetryEnabled: false" >> $soc_pillar_file + fi } telegraf_pillar() { diff --git a/setup/so-setup b/setup/so-setup index 2f62dca78..fc13e5b18 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -447,6 +447,7 @@ if ! [[ -f $install_opt_file ]]; then get_redirect # Does the user want to allow access to the UI? collect_so_allow + whiptail_accept_telemetry whiptail_end_settings elif [[ $is_standalone ]]; then waitforstate=true @@ -468,6 +469,7 @@ if ! [[ -f $install_opt_file ]]; then collect_webuser_inputs get_redirect collect_so_allow + whiptail_accept_telemetry whiptail_end_settings elif [[ $is_manager ]]; then info "Setting up as node type manager" @@ -488,6 +490,7 @@ if ! [[ -f $install_opt_file ]]; then collect_webuser_inputs get_redirect collect_so_allow + whiptail_accept_telemetry whiptail_end_settings elif [[ $is_managersearch ]]; then info "Setting up as node type managersearch" @@ -508,6 +511,7 @@ if ! [[ -f $install_opt_file ]]; then collect_webuser_inputs get_redirect collect_so_allow + whiptail_accept_telemetry whiptail_end_settings elif [[ $is_sensor ]]; then info "Setting up as node type sensor" @@ -597,6 +601,7 @@ if ! [[ -f $install_opt_file ]]; then collect_webuser_inputs get_redirect collect_so_allow + whiptail_accept_telemetry whiptail_end_settings elif [[ $is_receiver ]]; then diff --git a/setup/so-whiptail b/setup/so-whiptail index 904654c9b..95b21ccde 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -144,6 +144,26 @@ whiptail_cancel() { exit 1 } +whiptail_accept_telemetry() { + + [ -n "$TESTING" ] && return + + read -r -d '' message <<- EOM + + The Security Onion development team could use your help! Enabling SOC + Telemetry will help the team understand which UI features are being + used and enables informed prioritization of future development. + + Adjust this setting at anytime via the SOC Configuration screen. + + For more information visit https://docs.securityonion.net/telemetry.rst. + + Enable SOC Telemetry to help improve future releases? + EOM + whiptail --title "$whiptail_title" --yesno "$message" 13 75 + telemetry=$? +} + whiptail_check_exitstatus() { case $1 in 1)