From d501b0fac941a0e5245974e0bc419fe4e953f163 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Jan 2023 09:46:55 -0500 Subject: [PATCH 01/72] add elastic agent to assigned hostgroups --- salt/firewall/assigned_hostgroups.map.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index e91859743..6e249ec93 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -85,6 +85,8 @@ role: portgroups: - {{ portgroups.beats_5044 }} - {{ portgroups.beats_5644 }} + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} searchnodes: portgroups: - {{ portgroups.redis }} @@ -110,6 +112,10 @@ role: elasticsearch_rest: portgroups: - {{ portgroups.elasticsearch_rest }} + elastic_agent_endpoint: + portgroups: + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} endgame: portgroups: - {{ portgroups.endgame }} @@ -144,6 +150,8 @@ role: portgroups: - {{ portgroups.beats_5044 }} - {{ portgroups.beats_5644 }} + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} searchnodes: portgroups: - {{ portgroups.redis }} @@ -220,6 +228,8 @@ role: - {{ portgroups.yum }} - {{ portgroups.beats_5044 }} - {{ portgroups.beats_5644 }} + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} searchnodes: portgroups: - {{ portgroups.docker_registry }} From 1eafb8d62a2f84e51e78f8ead3f2e1a00a622daa Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Jan 2023 13:46:06 -0500 Subject: [PATCH 02/72] reload salt modules when docker is installed --- salt/docker/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/docker/init.sls b/salt/docker/init.sls index f8f89e058..b4a5506c9 100644 --- a/salt/docker/init.sls +++ b/salt/docker/init.sls @@ -19,6 +19,7 @@ dockerheldpackages: - docker-ce-rootless-extras: 20.10.5-3.el7 - hold: True - update_holds: True + - reload_modules: True #disable docker from managing iptables iptables_disabled: From a1fa4e3ef25c9b940662137b21f8bda8a760d61a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 20 Jan 2023 15:43:57 -0500 Subject: [PATCH 03/72] revert reload_modules since bugged --- salt/docker/init.sls | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/docker/init.sls b/salt/docker/init.sls index b4a5506c9..f8f89e058 100644 --- a/salt/docker/init.sls +++ b/salt/docker/init.sls @@ -19,7 +19,6 @@ dockerheldpackages: - docker-ce-rootless-extras: 20.10.5-3.el7 - hold: True - update_holds: True - - reload_modules: True #disable docker from managing iptables iptables_disabled: From f077b5c96df4babb14c6d0ef700621cf87b72abe Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 23 Jan 2023 13:11:50 -0500 Subject: [PATCH 04/72] Remove 18.04 --- setup/so-functions | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index a046b49ca..0005e90b4 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -909,9 +909,7 @@ detect_os() { elif [ -f /etc/os-release ]; then OS=ubuntu - if grep -q "UBUNTU_CODENAME=bionic" /etc/os-release; then - OSVER=bionic - elif grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then + if grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then OSVER=focal else info "We do not support your current version of Ubuntu." From d421aa82a20f74ba9a6005ceb1712b200e08f6a6 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 23 Jan 2023 14:04:26 -0500 Subject: [PATCH 05/72] do not treat all installs as ISO; fix check for non-empty mail files --- setup/so-verify | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-verify b/setup/so-verify index 685e50d81..77cdf7c8e 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -17,7 +17,7 @@ using_iso() { if [ "$setup_type" == "iso" ]; then return 0 fi - return 0 + return 1 } # Check entire setup log for errors or unexpected salt states @@ -54,7 +54,8 @@ log_has_errors() { # For ISO installs, we know nothing else can be running on this server, so there should be # nothing in any mail spool dir. cron_error_in_mail_spool() { - if find /var/spool/mail/ -type f -size +0 &> /dev/null; then + count=$(find /var/spool/mail/ -type f -size +0 | wc -l) + if [[ $count -ne 0 ]]; then return 0 fi return 1 From 22fbb953ea34bf038d8d426c1f59788bfe4bf75c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 23 Jan 2023 14:46:26 -0500 Subject: [PATCH 06/72] create cronjob to run highstate after setup --- salt/common/init.sls | 5 +++++ salt/setup/highstate_cron.sls | 6 ++++++ setup/so-setup | 1 + 3 files changed, 12 insertions(+) create mode 100644 salt/setup/highstate_cron.sls diff --git a/salt/common/init.sls b/salt/common/init.sls index a03eaacde..ea943ce4b 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -218,6 +218,11 @@ sostatus_check_cron: - month: '*' - dayweek: '*' +remove_post_setup_cron: + cron.absent: + - name: 'salt-call state.highstate' + - identifier: post_setup_cron + {% if GLOBALS.role not in ['eval', 'manager', 'managersearch', 'standalone'] %} soversionfile: diff --git a/salt/setup/highstate_cron.sls b/salt/setup/highstate_cron.sls new file mode 100644 index 000000000..dcbab4d8d --- /dev/null +++ b/salt/setup/highstate_cron.sls @@ -0,0 +1,6 @@ +post_setup_cron: + cron.present: + - name: 'salt-call state.highstate' + - user: root + - minute: '*/1' + - identifier: post_setup_cron diff --git a/setup/so-setup b/setup/so-setup index 99e218a61..58573dcdd 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -608,6 +608,7 @@ if ! [[ -f $install_opt_file ]]; then configure_minion "$minion_type" drop_install_options checkin_at_boot + logCmd "salt-call state.apply setup.highstate_cron --local --file-root=../salt/" whiptail_setup_complete fi From 5a016312f695b61ad1320ca1b52506afb6864a9f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 23 Jan 2023 14:53:39 -0500 Subject: [PATCH 07/72] switch MySQL 8 to use native password to avoid playbook incompatibility --- salt/mysql/etc/my.cnf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/mysql/etc/my.cnf b/salt/mysql/etc/my.cnf index e37f690dc..243cc4c51 100644 --- a/salt/mysql/etc/my.cnf +++ b/salt/mysql/etc/my.cnf @@ -29,3 +29,6 @@ symbolic-links=0 log-error=/var/log/mysql/mysqld.log pid-file=/var/run/mysqld/mysqld.pid + +# Switch back to the native password module so that playbook can connect +default_authentication_plugin=mysql_native_password \ No newline at end of file From d1ec7c8acee9fed2543a9bba573c1aeb296a9de5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 23 Jan 2023 15:50:53 -0500 Subject: [PATCH 08/72] remove to match with 2.4/dev --- setup/so-setup | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 5cbee5d9f..5088395f3 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -609,7 +609,6 @@ if ! [[ -f $install_opt_file ]]; then drop_install_options checkin_at_boot logCmd "salt-call state.apply setup.highstate_cron --local --file-root=../salt/" - whiptail_setup_complete verify_setup fi From d342f3c4b878d17082e066be3926556b6cabd9a8 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 23 Jan 2023 21:38:13 +0000 Subject: [PATCH 09/72] Add 'so-elastic-fleet-integration-policy-bulk-delete' to perform bulk deletion of integration policies --- ...-elastic-fleet-integration-policy-bulk-delete | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete new file mode 100644 index 000000000..e2c3095d6 --- /dev/null +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete @@ -0,0 +1,16 @@ +#/bin/bash +. /usr/sbin/so-common + +POLICY_ID=$1 + +# Let's snag a cookie from Kibana +SESSIONCOOKIE=$(curl -q -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') + +# Get integration policies relative to agent policy +INTEGRATION_POLICY_IDS=$(curl -q -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$POLICY_ID" | jq -r '.item.package_policies[].id') + +for i in $INTEGRATION_POLICY_IDS; do + # Delete integration policies + echo "Deleting integration policy: $i..." + curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d"{\"packagePolicyIds\": [\"$i\"], \"force\":true}"; +done From 40c6b380df1f816d5d80012ab3377642f7e563f9 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 23 Jan 2023 21:44:46 +0000 Subject: [PATCH 10/72] Update Import and Zeek integration policies; also update Zeek ingest node pipelines to set event.dataset. --- .../so-elastic-fleet-integration-policy-load | 44 +++++++------------ salt/elasticsearch/files/ingest/zeek.bacnet | 1 + .../files/ingest/zeek.bacnet_discovery | 1 + .../files/ingest/zeek.bacnet_property | 1 + .../files/ingest/zeek.bsap_ip_header | 1 + .../files/ingest/zeek.bsap_ip_rdb | 1 + .../files/ingest/zeek.bsap_ip_unknown | 1 + .../files/ingest/zeek.bsap_serial_header | 1 + .../files/ingest/zeek.bsap_serial_rdb | 1 + .../files/ingest/zeek.bsap_serial_rdb_ext | 1 + .../files/ingest/zeek.bsap_serial_unknown | 1 + salt/elasticsearch/files/ingest/zeek.cip | 1 + .../files/ingest/zeek.cip_identity | 1 + salt/elasticsearch/files/ingest/zeek.cip_io | 1 + salt/elasticsearch/files/ingest/zeek.conn | 1 + salt/elasticsearch/files/ingest/zeek.cotp | 1 + salt/elasticsearch/files/ingest/zeek.dce_rpc | 1 + salt/elasticsearch/files/ingest/zeek.dhcp | 1 + salt/elasticsearch/files/ingest/zeek.dnp3 | 1 + .../files/ingest/zeek.dnp3_control | 1 + .../files/ingest/zeek.dnp3_objects | 1 + salt/elasticsearch/files/ingest/zeek.dns | 1 + salt/elasticsearch/files/ingest/zeek.dpd | 1 + .../files/ingest/zeek.ecat_aoe_info | 1 + .../files/ingest/zeek.ecat_arp_info | 1 + .../files/ingest/zeek.ecat_coe_info | 1 + .../files/ingest/zeek.ecat_dev_info | 1 + .../files/ingest/zeek.ecat_foe_info | 1 + .../files/ingest/zeek.ecat_log_address | 1 + .../files/ingest/zeek.ecat_registers | 1 + .../files/ingest/zeek.ecat_soe_info | 1 + salt/elasticsearch/files/ingest/zeek.enip | 1 + salt/elasticsearch/files/ingest/zeek.files | 1 + salt/elasticsearch/files/ingest/zeek.ftp | 1 + salt/elasticsearch/files/ingest/zeek.http | 1 + salt/elasticsearch/files/ingest/zeek.intel | 1 + salt/elasticsearch/files/ingest/zeek.irc | 1 + salt/elasticsearch/files/ingest/zeek.kerberos | 1 + salt/elasticsearch/files/ingest/zeek.modbus | 1 + .../files/ingest/zeek.modbus_detailed | 1 + .../ingest/zeek.modbus_mask_write_register | 1 + .../zeek.modbus_read_write_multiple_registers | 1 + salt/elasticsearch/files/ingest/zeek.mysql | 1 + salt/elasticsearch/files/ingest/zeek.notice | 1 + salt/elasticsearch/files/ingest/zeek.ntlm | 1 + .../files/ingest/zeek.opcua_binary | 1 + .../ingest/zeek.opcua_binary_activate_session | 1 + ...nary_activate_session_client_software_cert | 1 + ...ua_binary_activate_session_diagnostic_info | 1 + ...ek.opcua_binary_activate_session_locale_id | 1 + .../files/ingest/zeek.opcua_binary_browse | 1 + .../zeek.opcua_binary_browse_description | 1 + .../zeek.opcua_binary_browse_diagnostic_info | 1 + ...a_binary_browse_request_continuation_point | 1 + ...ek.opcua_binary_browse_response_references | 1 + .../ingest/zeek.opcua_binary_browse_result | 1 + .../ingest/zeek.opcua_binary_create_session | 1 + ...zeek.opcua_binary_create_session_discovery | 1 + ...zeek.opcua_binary_create_session_endpoints | 1 + ...eek.opcua_binary_create_session_user_token | 1 + .../zeek.opcua_binary_create_subscription | 1 + .../ingest/zeek.opcua_binary_diag_info_detail | 1 + .../ingest/zeek.opcua_binary_get_endpoints | 1 + ...eek.opcua_binary_get_endpoints_description | 1 + .../zeek.opcua_binary_get_endpoints_discovery | 1 + .../zeek.opcua_binary_get_endpoints_locale_id | 1 + ...eek.opcua_binary_get_endpoints_profile_uri | 1 + ...zeek.opcua_binary_get_endpoints_user_token | 1 + .../zeek.opcua_binary_opensecure_channel | 1 + .../files/ingest/zeek.opcua_binary_read | 1 + .../ingest/zeek.opcua_binary_read_array_dims | 1 + .../zeek.opcua_binary_read_array_dims_link | 1 + .../zeek.opcua_binary_read_diagnostic_info | 1 + .../zeek.opcua_binary_read_extension_object | 1 + ...ek.opcua_binary_read_extension_object_link | 1 + .../zeek.opcua_binary_read_nodes_to_read | 1 + .../ingest/zeek.opcua_binary_read_results | 1 + .../zeek.opcua_binary_read_results_link | 1 + .../ingest/zeek.opcua_binary_read_status_code | 1 + .../zeek.opcua_binary_read_variant_data | 1 + .../zeek.opcua_binary_read_variant_data_link | 1 + .../zeek.opcua_binary_status_code_detail | 1 + salt/elasticsearch/files/ingest/zeek.pe | 1 + salt/elasticsearch/files/ingest/zeek.profinet | 1 + .../files/ingest/zeek.profinet_dce_rpc | 1 + salt/elasticsearch/files/ingest/zeek.radius | 1 + salt/elasticsearch/files/ingest/zeek.rdp | 1 + salt/elasticsearch/files/ingest/zeek.rfb | 1 + salt/elasticsearch/files/ingest/zeek.s7comm | 1 + .../files/ingest/zeek.s7comm_plus | 1 + .../files/ingest/zeek.s7comm_read_szl | 1 + .../files/ingest/zeek.s7comm_upload_download | 1 + .../files/ingest/zeek.signatures | 1 + salt/elasticsearch/files/ingest/zeek.sip | 1 + .../elasticsearch/files/ingest/zeek.smb_files | 1 + .../files/ingest/zeek.smb_mapping | 1 + salt/elasticsearch/files/ingest/zeek.smtp | 1 + salt/elasticsearch/files/ingest/zeek.snmp | 1 + salt/elasticsearch/files/ingest/zeek.socks | 1 + salt/elasticsearch/files/ingest/zeek.software | 1 + salt/elasticsearch/files/ingest/zeek.ssh | 1 + salt/elasticsearch/files/ingest/zeek.ssl | 1 + salt/elasticsearch/files/ingest/zeek.stun | 1 + salt/elasticsearch/files/ingest/zeek.stun_nat | 1 + salt/elasticsearch/files/ingest/zeek.syslog | 1 + salt/elasticsearch/files/ingest/zeek.tds | 1 + salt/elasticsearch/files/ingest/zeek.tds_rpc | 1 + .../files/ingest/zeek.tds_sql_batch | 1 + salt/elasticsearch/files/ingest/zeek.tunnel | 1 + salt/elasticsearch/files/ingest/zeek.tunnels | 1 + salt/elasticsearch/files/ingest/zeek.weird | 1 + .../elasticsearch/files/ingest/zeek.wireguard | 1 + salt/elasticsearch/files/ingest/zeek.x509 | 1 + 113 files changed, 129 insertions(+), 27 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index 614fc81b1..1a5523723 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -18,58 +18,47 @@ echo "Setting up default Security Onion package policies for Elastic Agent..." # Set up Suricata logs echo "Setting up Suricata package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "suricata-logs", "name": "suricata-logs", "description": "Suricata integration", "namespace": "suricata_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/suricata/eve*.json" ], "data_stream.dataset": "alert", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata", "custom": "pipeline: suricata.common" }}}}}}' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "suricata-logs", "name": "suricata-logs", "description": "Suricata integration", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/suricata/eve*.json" ], "data_stream.dataset": "suricata", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata", "custom": "pipeline: suricata.common" }}}}}}' + +# Set up Zeek logs +echo "Setting up Zeek package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "zeek-logs", "description": "Zeek logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/zeek/logs/current/*.log"], "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek", "custom": "prospector.scanner.exclude_files: [\"(broker | capture_loss | loaded_scripts | packet_filter | stats | stderr | stdout).log$\"]\n" } } } } } }' # Import - Suricata echo "Settings up Suricata import package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-suricata-logs", "name": "import-suricata-logs", "description": "Import Suricata logs", "namespace": "import_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/suricata/eve*.json"], "data_stream.dataset": "alert", "tags": ["import"], "processors": "- add_fields:\n target: event\n fields:\n category: file\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"", "custom": "pipeline: suricata.common" } } } } } }' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-suricata-logs", "name": "import-suricata-logs", "description": "Import Suricata logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/suricata/eve*.json"], "data_stream.dataset": "import", "tags": ["import"], "processors": "- add_fields:\n target: event\n fields:\n category: file\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"", "custom": "pipeline: suricata.common" } } } } } }' -# Zeek logs -{%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} - {%- if ZEEKVER != 'SURICATA' %} - {% import_yaml 'filebeat/defaults.yaml' as FBD with context %} - {% set FBCONFIG = salt['pillar.get']('filebeat:zeek_logs_enabled', default=FBD.filebeat, merge=True) %} - {%- for LOGNAME in FBCONFIG.zeek_logs_enabled %} - -# Set up Zeek {{ LOGNAME }} logs -echo "Setting up Zeek {{ LOGNAME }} package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "zeek-{{ LOGNAME }}", "description": "Zeek {{ LOGNAME }} logs", "namespace": "zeek_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/zeek/logs/current/{{ LOGNAME }}.log" ], "data_stream.dataset": "{{ LOGNAME }}", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: zeek", "custom": "pipeline: zeek.{{ LOGNAME }}" }}}}}}' -echo - -echo "Setting up Zeek import {{ LOGNAME }} package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-zeek-{{ LOGNAME }}-logs", "name": "import-zeek-{{ LOGNAME }}-logs", "description": "Import Zeek {{ LOGNAME }} logs", "namespace": "import_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/zeek/logs/{{ LOGNAME }}.log"], "data_stream.dataset": "{{ LOGNAME }}", "tags": ["import"], "processors": "- add_fields:\n target: event\n fields:\n category: file\n module: suricata\n imported: true", "custom": "pipeline: zeek.{{ LOGNAME }}" } } } } } }' - - {%- endfor %} - {%- endif %} -{%- endif %} +# Set Import - Zeek logs +echo "Setting up Zeek Import package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "import-zeek-logs", "description": "Zeek Import logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/zeek/logs/*.log"], "data_stream.dataset": "import", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n imported: true", "custom": "prospector.scanner.exclude_files: [\"(broker | capture_loss | loaded_scripts | packet_filter | stats | stderr | stdout).log$\"]" } } } } } }' # Strelka logs echo "Setting up Strelka package policy" -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "strelka-logs", "name": "strelka-logs", "description": "Strelka logs", "namespace": "strelka_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], "data_stream.dataset": "file", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: file\n module: strelka", "custom": "pipeline: strelka.file" }}}}}}' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "strelka-logs", "name": "strelka-logs", "description": "Strelka logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], "data_stream.dataset": "file", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: file\n module: strelka", "custom": "pipeline: strelka.file" }}}}}}' # Syslog TCP Port 514 echo "Setting up Syslog TCP package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "tcp", "version": "1.5.0" }, "id": "syslog-tcp-514", "name": "syslog-tcp-514", "description": "Syslog Over TCP Port 514", "namespace": "syslog_so", "inputs": { "tcp-tcp": { "enabled": true, "streams": { "tcp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "processors": "- add_fields:\n target: event\n fields:\n module: syslog", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "tcp", "version": "1.5.0" }, "id": "syslog-tcp-514", "name": "syslog-tcp-514", "description": "Syslog Over TCP Port 514", "namespace": "so", "inputs": { "tcp-tcp": { "enabled": true, "streams": { "tcp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "processors": "- add_fields:\n target: event\n fields:\n module: syslog", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' # Syslog UDP Port 514 echo "Setting up Syslog UDP package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "udp", "version": "1.5.0" }, "id": "syslog-udp-514", "name": "syslog-udp-514", "description": "Syslog over UDP Port 514", "namespace": "syslog_so", "inputs": { "udp-udp": { "enabled": true, "streams": { "udp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "max_message_size": "10KiB", "keep_null": false, "processors": "- add_fields:\n target: event\n fields: \n module: syslog\n", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "udp", "version": "1.5.0" }, "id": "syslog-udp-514", "name": "syslog-udp-514", "description": "Syslog over UDP Port 514", "namespace": "so", "inputs": { "udp-udp": { "enabled": true, "streams": { "udp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "max_message_size": "10KiB", "keep_null": false, "processors": "- add_fields:\n target: event\n fields: \n module: syslog\n", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' # Kratos logs echo "Setting up Kratos package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "kratos_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "access", "tags": [],"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- rename:\n fields:\n - from: \"audience\"\n to: \"event.dataset\"\n ignore_missing: true\n- add_fields:\n when:\n not: \n has_fields: ['event.dataset']\n target: ''\n fields:\n event.dataset: access", "custom": "pipeline: kratos" }}}}}}' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "access", "tags": [],"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- rename:\n fields:\n - from: \"audience\"\n to: \"event.dataset\"\n ignore_missing: true\n- add_fields:\n when:\n not: \n has_fields: ['event.dataset']\n target: ''\n fields:\n event.dataset: access", "custom": "pipeline: kratos" }}}}}}' # RITA Beacon logs echo "Setting up RITA Beacon package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-beacon-logs", "name": "rita-beacon-logs", "description": "RITA Beacon logs", "namespace": "rita_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv" ], "data_stream.dataset": "beacon", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.beacon" }}}}}}' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-beacon-logs", "name": "rita-beacon-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv" ], "data_stream.dataset": "beacon", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.beacon" }}}}}}' # RITA Connection Logs echo "Setting up RITA connection package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-connection-logs", "name": "rita-connection-logs", "description": "RITA Connection logs", "namespace": "rita_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/long-connections.csv" ], "data_stream.dataset": "connections", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.connections" }}}}}}' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-connection-logs", "name": "rita-connection-logs", "description": "RITA Connection logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/long-connections.csv" ], "data_stream.dataset": "connections", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.connections" }}}}}}' # RITA DNS Logs echo "Setting up RITA DNS package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-exploded-dns-logs", "name": "rita-exploded-dns-logs", "description": "RITA Exploded DNS logs", "namespace": "rita_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "exploded_dns", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.dns" }}}}}}' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-exploded-dns-logs", "name": "rita-exploded-dns-logs", "description": "RITA Exploded DNS logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "exploded_dns", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.dns" }}}}}}' # Elasticsearch logs curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "elasticsearch", "version": "1.0.0" }, "id": "elasticsearch-logs", "name": "elasticsearch-logs", "description": "Elasticsearch Logs", "namespace": "default", "inputs": { "elasticsearch-logfile": { "enabled": true, "streams": { "elasticsearch.audit": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_audit.json" ] } }, "elasticsearch.deprecation": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_deprecation.json" ] } }, "elasticsearch.gc": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/gc.log.[0-9]*", "/var/log/elasticsearch/gc.log" ] } }, "elasticsearch.server": { "enabled": true, "vars": { "paths": [ "/opt/so/log/elasticsearch/*.log" ] } }, "elasticsearch.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_index_search_slowlog.json", "/var/log/elasticsearch/*_index_indexing_slowlog.json" ] } } } }, "elasticsearch-elasticsearch/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9200" ], "scope": "node" }, "streams": { "elasticsearch.stack_monitoring.ccr": { "enabled": false }, "elasticsearch.stack_monitoring.cluster_stats": { "enabled": false }, "elasticsearch.stack_monitoring.enrich": { "enabled": false }, "elasticsearch.stack_monitoring.index": { "enabled": false }, "elasticsearch.stack_monitoring.index_recovery": { "enabled": false, "vars": { "active.only": true } }, "elasticsearch.stack_monitoring.index_summary": { "enabled": false }, "elasticsearch.stack_monitoring.ml_job": { "enabled": false }, "elasticsearch.stack_monitoring.node": { "enabled": false }, "elasticsearch.stack_monitoring.node_stats": { "enabled": false }, "elasticsearch.stack_monitoring.pending_tasks": { "enabled": false }, "elasticsearch.stack_monitoring.shard": { "enabled": false } } } } }' @@ -82,3 +71,4 @@ curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POS # Redis logs curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "redis", "version": "1.4.0" }, "id": "redis-logs", "name": "redis-logs", "description": "Redis logs", "namespace": "default", "inputs": { "redis-logfile": { "enabled": true, "streams": { "redis.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/redis/redis.log" ], "tags": [ "redis-log" ], "preserve_original_event": false } } } }, "redis-redis": { "enabled": false, "streams": { "redis.slowlog": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "password": "" } } } }, "redis-redis/metrics": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "idle_timeout": "20s", "maxconn": 10, "network": "tcp", "password": "" }, "streams": { "redis.info": { "enabled": false, "vars": { "period": "10s" } }, "redis.key": { "enabled": false, "vars": { "key.patterns": "- limit: 20\n pattern: '*'\n", "period": "10s" } }, "redis.keyspace": { "enabled": false, "vars": { "period": "10s" } } } } } }' + diff --git a/salt/elasticsearch/files/ingest/zeek.bacnet b/salt/elasticsearch/files/ingest/zeek.bacnet index a96a05b56..08e89d7a3 100644 --- a/salt/elasticsearch/files/ingest/zeek.bacnet +++ b/salt/elasticsearch/files/ingest/zeek.bacnet @@ -1,6 +1,7 @@ { "description" : "zeek.bacnet", "processors" : [ + { "set": { "field": "event.dataset", "value": "bacnet" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.is_orig", "target_field": "bacnet.is_orig", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bacnet_discovery b/salt/elasticsearch/files/ingest/zeek.bacnet_discovery index c51cbf5c1..f848e683a 100644 --- a/salt/elasticsearch/files/ingest/zeek.bacnet_discovery +++ b/salt/elasticsearch/files/ingest/zeek.bacnet_discovery @@ -1,6 +1,7 @@ { "description" : "zeek.bacnet_discovery", "processors" : [ + { "set": { "field": "event.dataset", "value": "bacnet_discovery" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.is_orig", "target_field": "bacnet.is_orig", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bacnet_property b/salt/elasticsearch/files/ingest/zeek.bacnet_property index d04c4c327..80e23652f 100644 --- a/salt/elasticsearch/files/ingest/zeek.bacnet_property +++ b/salt/elasticsearch/files/ingest/zeek.bacnet_property @@ -1,6 +1,7 @@ { "description" : "zeek.bacnet_property", "processors" : [ + { "set": { "field": "event.dataset", "value": "bacnet_property" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.is_orig", "target_field": "bacnet.is_orig", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bsap_ip_header b/salt/elasticsearch/files/ingest/zeek.bsap_ip_header index b92d7f233..965d642df 100644 --- a/salt/elasticsearch/files/ingest/zeek.bsap_ip_header +++ b/salt/elasticsearch/files/ingest/zeek.bsap_ip_header @@ -1,6 +1,7 @@ { "description" : "zeek.bsap_ip_header", "processors" : [ + { "set": { "field": "event.dataset", "value": "bsap_ip_header" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.num_msg", "target_field": "bsap.number.messages", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bsap_ip_rdb b/salt/elasticsearch/files/ingest/zeek.bsap_ip_rdb index f5ebd3a0a..aade31c21 100644 --- a/salt/elasticsearch/files/ingest/zeek.bsap_ip_rdb +++ b/salt/elasticsearch/files/ingest/zeek.bsap_ip_rdb @@ -1,6 +1,7 @@ { "description" : "zeek.bsap_ip_rdb", "processors" : [ + { "set": { "field": "event.dataset", "value": "bsap_ip_rdb" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.header_size", "target_field": "bsap.header.length", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bsap_ip_unknown b/salt/elasticsearch/files/ingest/zeek.bsap_ip_unknown index 15d9a8fd9..d561c90b0 100644 --- a/salt/elasticsearch/files/ingest/zeek.bsap_ip_unknown +++ b/salt/elasticsearch/files/ingest/zeek.bsap_ip_unknown @@ -1,6 +1,7 @@ { "description" : "zeek.bsap_ip_unknown", "processors" : [ + { "set": { "field": "event.dataset", "value": "bsap_ip_unknown" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.data", "target_field": "bsap.ip.unknown.data", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bsap_serial_header b/salt/elasticsearch/files/ingest/zeek.bsap_serial_header index b4b499ff1..2cfcd7ecf 100644 --- a/salt/elasticsearch/files/ingest/zeek.bsap_serial_header +++ b/salt/elasticsearch/files/ingest/zeek.bsap_serial_header @@ -1,6 +1,7 @@ { "description" : "zeek.bsap_serial_header", "processors" : [ + { "set": { "field": "event.dataset", "value": "bsap_serial_header" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.ser", "target_field": "bsap.message.serial_number", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb b/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb index f45d8c0cd..8e7f07802 100644 --- a/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb +++ b/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb @@ -1,6 +1,7 @@ { "description" : "zeek.bsap_serial_rdb", "processors" : [ + { "set": { "field": "event.dataset", "value": "bsap_serial_rdb" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.func_code", "target_field": "bsap.rdb.function", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb_ext b/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb_ext index 2ca85ad78..110095b59 100644 --- a/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb_ext +++ b/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb_ext @@ -1,6 +1,7 @@ { "description" : "zeek.bsap_serial_rdb_ext", "processors" : [ + { "set": { "field": "event.dataset", "value": "bsap_serial_rdb_ext" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.dfun", "target_field": "bsap.destination.function", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bsap_serial_unknown b/salt/elasticsearch/files/ingest/zeek.bsap_serial_unknown index ea0c5e471..bb30c5940 100644 --- a/salt/elasticsearch/files/ingest/zeek.bsap_serial_unknown +++ b/salt/elasticsearch/files/ingest/zeek.bsap_serial_unknown @@ -1,6 +1,7 @@ { "description" : "zeek.bsap_serial_unknown", "processors" : [ + { "set": { "field": "event.dataset", "value": "bsap_serial_unknown" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.data", "target_field": "bsap.serial.unknown.data", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.cip b/salt/elasticsearch/files/ingest/zeek.cip index 5182a7037..a704c2b28 100644 --- a/salt/elasticsearch/files/ingest/zeek.cip +++ b/salt/elasticsearch/files/ingest/zeek.cip @@ -1,6 +1,7 @@ { "description" : "zeek.cip", "processors" : [ + { "set": { "field": "event.dataset", "value": "cip" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.is_orig", "target_field": "cip.is_orig", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.cip_identity b/salt/elasticsearch/files/ingest/zeek.cip_identity index a3522f86a..e36678f66 100644 --- a/salt/elasticsearch/files/ingest/zeek.cip_identity +++ b/salt/elasticsearch/files/ingest/zeek.cip_identity @@ -1,6 +1,7 @@ { "description" : "zeek.cip_identity", "processors" : [ + { "set": { "field": "event.dataset", "value": "cip_identity" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.encapsulation_version", "target_field": "cip.encapsulation.version", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.cip_io b/salt/elasticsearch/files/ingest/zeek.cip_io index 68c376b05..f41c3f431 100644 --- a/salt/elasticsearch/files/ingest/zeek.cip_io +++ b/salt/elasticsearch/files/ingest/zeek.cip_io @@ -1,6 +1,7 @@ { "description" : "zeek.cip_io", "processors" : [ + { "set": { "field": "event.dataset", "value": "cip_io" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.is_orig", "target_field": "cip.is_orig", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.conn b/salt/elasticsearch/files/ingest/zeek.conn index 14689f782..73ce29c1d 100644 --- a/salt/elasticsearch/files/ingest/zeek.conn +++ b/salt/elasticsearch/files/ingest/zeek.conn @@ -1,6 +1,7 @@ { "description" : "zeek.conn", "processors" : [ + { "set": { "field": "event.dataset", "value": "conn" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.cotp b/salt/elasticsearch/files/ingest/zeek.cotp index fb4b090cd..f5abd87f6 100644 --- a/salt/elasticsearch/files/ingest/zeek.cotp +++ b/salt/elasticsearch/files/ingest/zeek.cotp @@ -1,6 +1,7 @@ { "description" : "zeek.cotp", "processors" : [ + { "set": { "field": "event.dataset", "value": "cotp" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.pdu_code", "target_field": "cotp.pdu.code", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.dce_rpc b/salt/elasticsearch/files/ingest/zeek.dce_rpc index 247946073..0f6ae6d67 100644 --- a/salt/elasticsearch/files/ingest/zeek.dce_rpc +++ b/salt/elasticsearch/files/ingest/zeek.dce_rpc @@ -1,6 +1,7 @@ { "description" : "zeek.dce_rpc", "processors" : [ + { "set": { "field": "event.dataset", "value": "dce_rpc" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.rtt", "target_field": "event.duration", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.dhcp b/salt/elasticsearch/files/ingest/zeek.dhcp index f3a551edd..d2d151847 100644 --- a/salt/elasticsearch/files/ingest/zeek.dhcp +++ b/salt/elasticsearch/files/ingest/zeek.dhcp @@ -1,6 +1,7 @@ { "description" : "zeek.dhcp", "processors" : [ + { "set": { "field": "event.dataset", "value": "dhcp" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.uids", "target_field": "log.id.uids", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.dnp3 b/salt/elasticsearch/files/ingest/zeek.dnp3 index 6618289f8..e8011463b 100644 --- a/salt/elasticsearch/files/ingest/zeek.dnp3 +++ b/salt/elasticsearch/files/ingest/zeek.dnp3 @@ -1,6 +1,7 @@ { "description" : "zeek.dnp3", "processors" : [ + { "set": { "field": "event.dataset", "value": "dnp3" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.fc_request", "target_field": "dnp3.fc_request", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.dnp3_control b/salt/elasticsearch/files/ingest/zeek.dnp3_control index 0c465b5d2..e6a3a106c 100644 --- a/salt/elasticsearch/files/ingest/zeek.dnp3_control +++ b/salt/elasticsearch/files/ingest/zeek.dnp3_control @@ -1,6 +1,7 @@ { "description" : "zeek.dnp3_control", "processors" : [ + { "set": { "field": "event.dataset", "value": "dnp3_control" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.block_type", "target_field": "dnp3.block_type", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.dnp3_objects b/salt/elasticsearch/files/ingest/zeek.dnp3_objects index c78ae9e1f..bdce7e8d1 100644 --- a/salt/elasticsearch/files/ingest/zeek.dnp3_objects +++ b/salt/elasticsearch/files/ingest/zeek.dnp3_objects @@ -1,6 +1,7 @@ { "description" : "zeek.dnp3_objects", "processors" : [ + { "set": { "field": "event.dataset", "value": "dnp3_objects" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.function_code", "target_field": "dnp3.function_code", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.dns b/salt/elasticsearch/files/ingest/zeek.dns index 9d7f7e93b..5f39b8017 100644 --- a/salt/elasticsearch/files/ingest/zeek.dns +++ b/salt/elasticsearch/files/ingest/zeek.dns @@ -1,6 +1,7 @@ { "description" : "zeek.dns", "processors" : [ + { "set": { "field": "event.dataset", "value": "dns" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.dpd b/salt/elasticsearch/files/ingest/zeek.dpd index 59ceed368..2f76c5ecb 100644 --- a/salt/elasticsearch/files/ingest/zeek.dpd +++ b/salt/elasticsearch/files/ingest/zeek.dpd @@ -1,6 +1,7 @@ { "description" : "zeek.dpd", "processors" : [ + { "set": { "field": "event.dataset", "value": "dpd" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_aoe_info b/salt/elasticsearch/files/ingest/zeek.ecat_aoe_info index c5f9b9dc3..e9c056666 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_aoe_info +++ b/salt/elasticsearch/files/ingest/zeek.ecat_aoe_info @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_aoe_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_aoe_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.targetid", "target_field": "destination.mac", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_arp_info b/salt/elasticsearch/files/ingest/zeek.ecat_arp_info index cbc3676ab..750217dd3 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_arp_info +++ b/salt/elasticsearch/files/ingest/zeek.ecat_arp_info @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_arp_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_arp_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.arp_type", "target_field": "ecat.arp.type", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_coe_info b/salt/elasticsearch/files/ingest/zeek.ecat_coe_info index e425e3173..8571ece05 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_coe_info +++ b/salt/elasticsearch/files/ingest/zeek.ecat_coe_info @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_coe_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_coe_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.number", "target_field": "ecat.message.number", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_dev_info b/salt/elasticsearch/files/ingest/zeek.ecat_dev_info index d01289e9a..165c04134 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_dev_info +++ b/salt/elasticsearch/files/ingest/zeek.ecat_dev_info @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_dev_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_dev_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.slave_id", "target_field": "ecat.slave.address", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_foe_info b/salt/elasticsearch/files/ingest/zeek.ecat_foe_info index 6655f2cb7..810435827 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_foe_info +++ b/salt/elasticsearch/files/ingest/zeek.ecat_foe_info @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_foe_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_foe_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcode", "target_field": "ecat.operation.code", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_log_address b/salt/elasticsearch/files/ingest/zeek.ecat_log_address index ad0ee161f..c3f15ca29 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_log_address +++ b/salt/elasticsearch/files/ingest/zeek.ecat_log_address @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_log_address", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_log_address" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.srcmac", "target_field": "source.mac", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_registers b/salt/elasticsearch/files/ingest/zeek.ecat_registers index d0a11ba83..363a71918 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_registers +++ b/salt/elasticsearch/files/ingest/zeek.ecat_registers @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_registers", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_registers" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.srcmac", "target_field": "source.mac", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_soe_info b/salt/elasticsearch/files/ingest/zeek.ecat_soe_info index bddc40efa..5785348bb 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_soe_info +++ b/salt/elasticsearch/files/ingest/zeek.ecat_soe_info @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_soe_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_soe_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcode", "target_field": "ecat.operation.code", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.enip b/salt/elasticsearch/files/ingest/zeek.enip index fee3b058b..443a72f9a 100644 --- a/salt/elasticsearch/files/ingest/zeek.enip +++ b/salt/elasticsearch/files/ingest/zeek.enip @@ -1,6 +1,7 @@ { "description" : "zeek.enip", "processors" : [ + { "set": { "field": "event.dataset", "value": "enip" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.is_orig", "target_field": "enip.is_orig", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.files b/salt/elasticsearch/files/ingest/zeek.files index c5fefa85f..105bbd326 100644 --- a/salt/elasticsearch/files/ingest/zeek.files +++ b/salt/elasticsearch/files/ingest/zeek.files @@ -1,6 +1,7 @@ { "description" : "zeek.files", "processors" : [ + { "set": { "field": "event.dataset", "value": "files" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.fuid", "target_field": "log.id.fuid", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ftp b/salt/elasticsearch/files/ingest/zeek.ftp index 45aa76055..3767c7b10 100644 --- a/salt/elasticsearch/files/ingest/zeek.ftp +++ b/salt/elasticsearch/files/ingest/zeek.ftp @@ -1,6 +1,7 @@ { "description" : "zeek.ftp", "processors" : [ + { "set": { "field": "event.dataset", "value": "ftp" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.user", "target_field": "ftp.user", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.http b/salt/elasticsearch/files/ingest/zeek.http index 3368e45e1..2224da2f7 100644 --- a/salt/elasticsearch/files/ingest/zeek.http +++ b/salt/elasticsearch/files/ingest/zeek.http @@ -1,6 +1,7 @@ { "description" : "zeek.http", "processors" : [ + { "set": { "field": "event.dataset", "value": "http" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.trans_depth", "target_field": "http.trans_depth", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.intel b/salt/elasticsearch/files/ingest/zeek.intel index 8be25c9ef..f3553ebd3 100644 --- a/salt/elasticsearch/files/ingest/zeek.intel +++ b/salt/elasticsearch/files/ingest/zeek.intel @@ -1,6 +1,7 @@ { "description" : "zeek.intel", "processors" : [ + { "set": { "field": "event.dataset", "value": "intel" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "dot_expander": { "field": "seen.indicator", "path": "message2", "ignore_failure": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.irc b/salt/elasticsearch/files/ingest/zeek.irc index 3d40d7f80..e21f36553 100644 --- a/salt/elasticsearch/files/ingest/zeek.irc +++ b/salt/elasticsearch/files/ingest/zeek.irc @@ -1,6 +1,7 @@ { "description" : "zeek.irc", "processors" : [ + { "set": { "field": "event.dataset", "value": "irc" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.nick", "target_field": "irc.nickname", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.kerberos b/salt/elasticsearch/files/ingest/zeek.kerberos index 0b09ebf61..f8aec42ec 100644 --- a/salt/elasticsearch/files/ingest/zeek.kerberos +++ b/salt/elasticsearch/files/ingest/zeek.kerberos @@ -1,6 +1,7 @@ { "description" : "zeek.kerberos", "processors" : [ + { "set": { "field": "event.dataset", "value": "kerberos" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.request_type", "target_field": "kerberos.request_type", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.modbus b/salt/elasticsearch/files/ingest/zeek.modbus index a9dd91430..92d012862 100644 --- a/salt/elasticsearch/files/ingest/zeek.modbus +++ b/salt/elasticsearch/files/ingest/zeek.modbus @@ -1,6 +1,7 @@ { "description" : "zeek.modbus", "processors" : [ + { "set": { "field": "event.dataset", "value": "modbus" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.func", "target_field": "modbus.function", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.modbus_detailed b/salt/elasticsearch/files/ingest/zeek.modbus_detailed index 635566c6a..f5bdf8200 100644 --- a/salt/elasticsearch/files/ingest/zeek.modbus_detailed +++ b/salt/elasticsearch/files/ingest/zeek.modbus_detailed @@ -1,6 +1,7 @@ { "description" : "zeek.modbus_detailed", "processors" : [ + { "set": { "field": "event.dataset", "value": "modbus_detailed" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.unit_id", "target_field": "modbus.unit_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.modbus_mask_write_register b/salt/elasticsearch/files/ingest/zeek.modbus_mask_write_register index d548fe615..9e4a0b21b 100644 --- a/salt/elasticsearch/files/ingest/zeek.modbus_mask_write_register +++ b/salt/elasticsearch/files/ingest/zeek.modbus_mask_write_register @@ -1,6 +1,7 @@ { "description" : "zeek.modbus_mask_write_register", "processors" : [ + { "set": { "field": "event.dataset", "value": "modbus_mask_write_register" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.unit_id", "target_field": "modbus.unit_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.modbus_read_write_multiple_registers b/salt/elasticsearch/files/ingest/zeek.modbus_read_write_multiple_registers index 234faa34f..2c7bf04b4 100644 --- a/salt/elasticsearch/files/ingest/zeek.modbus_read_write_multiple_registers +++ b/salt/elasticsearch/files/ingest/zeek.modbus_read_write_multiple_registers @@ -1,6 +1,7 @@ { "description" : "zeek.read_write_multiple_registers", "processors" : [ + { "set": { "field": "event.dataset", "value": "modbus_read_write_multiple_registers" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.unit_id", "target_field": "modbus.unit_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.mysql b/salt/elasticsearch/files/ingest/zeek.mysql index e950d5f64..3e57e2a29 100644 --- a/salt/elasticsearch/files/ingest/zeek.mysql +++ b/salt/elasticsearch/files/ingest/zeek.mysql @@ -1,6 +1,7 @@ { "description" : "zeek.mysql", "processors" : [ + { "set": { "field": "event.dataset", "value": "mysql" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.cmd", "target_field": "mysql.command", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.notice b/salt/elasticsearch/files/ingest/zeek.notice index b662393f6..8e086ad05 100644 --- a/salt/elasticsearch/files/ingest/zeek.notice +++ b/salt/elasticsearch/files/ingest/zeek.notice @@ -1,6 +1,7 @@ { "description" : "zeek.notice", "processors" : [ + { "set": { "field": "event.dataset", "value": "notice" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.fuid", "target_field": "log.id.fuid", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ntlm b/salt/elasticsearch/files/ingest/zeek.ntlm index c8f2e37c0..a7d0cf782 100644 --- a/salt/elasticsearch/files/ingest/zeek.ntlm +++ b/salt/elasticsearch/files/ingest/zeek.ntlm @@ -1,6 +1,7 @@ { "description" : "zeek.ntlm", "processors" : [ + { "set": { "field": "event.dataset", "value": "ntlm" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.hostname", "target_field": "host.name", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary b/salt/elasticsearch/files/ingest/zeek.opcua_binary index 37a9cdf1a..acb2e1b6b 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.msg_type", "target_field": "opcua.message_type", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session index 81f8e9392..00e3aa447 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_activate_session", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_activate_session" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true} }, { "rename": { "field": "message2.opcua_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_client_software_cert b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_client_software_cert index fe6f577eb..5f4f3d42d 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_client_software_cert +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_client_software_cert @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_activate_session_client_software_cert", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_activate_session_client_software_cert" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.client_software_cert_link_id", "target_field": "opcua.client_software_cert_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_diagnostic_info b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_diagnostic_info index 283d1c78e..5363fe73d 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_diagnostic_info +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_diagnostic_info @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_activate_session_diagnostic_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_activate_session_diagnostic_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.activate_session_diag_info_link_id", "target_field": "opcua.activate_session_diag_info_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_locale_id b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_locale_id index 08d8a672e..353047101 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_locale_id +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_locale_id @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_activate_session_locale_id", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_activate_session_locale_id" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcua_locale_link_id", "target_field": "opcua.locale_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse index 8c4d919cd..6bb11c636 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_browse", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_browse" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcua_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_description b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_description index f1439f192..df35dff9b 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_description +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_description @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_browse_description", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_browse_description" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.browse_description_link_id", "target_field": "opcua.browse_description_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_diagnostic_info b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_diagnostic_info index 43560f4f0..71060bb6b 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_diagnostic_info +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_diagnostic_info @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_browse_diagnostic_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_browse_diagnostic_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.browse_diag_info_link_id", "target_field": "opcua.browse_session_diag_info_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_request_continuation_point b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_request_continuation_point index ce971109b..48e68edad 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_request_continuation_point +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_request_continuation_point @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_browse_request_continuation_point", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_browse_request_continuation_point" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.browse_next_link_id", "target_field": "opcua.browse_next_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_response_references b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_response_references index 960a0a939..ccfff9911 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_response_references +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_response_references @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_browse_response_references", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_browse_response_references" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.browse_reference_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_result b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_result index d9eabbd87..9d47076fd 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_result +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_result @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_browse_result", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_browse_result" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.browse_response_link_id", "target_field": "opcua.response_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session index d7e0d3d87..d2e490f1f 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_create_session", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_create_session" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true} }, { "rename": { "field": "message2.opcua_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_discovery b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_discovery index cf9a56135..7a7753e45 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_discovery +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_discovery @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_create_session_discovery", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_create_session_discovery" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.discovery_profile_link_id", "target_field": "opcua.discovery_profile_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_endpoints b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_endpoints index 0511211d5..c1e599c1e 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_endpoints +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_endpoints @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_create_session_endpoints", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_create_session_endpoints" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.endpoint_link_id", "target_field": "opcua.endpoint_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_user_token b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_user_token index b86ec066d..169479a48 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_user_token +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_user_token @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_create_session_user_token", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_create_session_user_token" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.user_token_link_id", "target_field": "opcua.user_token_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_subscription b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_subscription index 832ac75b1..c282f9393 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_subscription +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_subscription @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_create_subscription", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_create_subscription" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcua_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_diag_info_detail b/salt/elasticsearch/files/ingest/zeek.opcua_binary_diag_info_detail index 170c35be0..9d6d63896 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_diag_info_detail +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_diag_info_detail @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_diag_info_detail", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_diag_info_detail" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.diag_info_link_id", "target_field": "opcua.diag_info_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints index 51f9349fc..90de3e82f 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_get_endpoints", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_get_endpoints" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcua_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_description b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_description index b467196de..de9c9b0bc 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_description +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_description @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_get_endpoints_description", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_get_endpoints_description" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.endpoint_description_link_id", "target_field": "opcua.endpoint_description_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_discovery b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_discovery index eeaf91dcb..4e9f964cf 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_discovery +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_discovery @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_get_endpoints_discovery", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_get_endpoints_discovery" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.discovery_profile_link_id", "target_field": "opcua.discovery_profile_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_locale_id b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_locale_id index 3716b3bb5..ea592e19e 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_locale_id +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_locale_id @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_get_endpoints_locale_id", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_get_endpoints_locale_id" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcua_locale_link_id", "target_field": "opcua.locale_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_profile_uri b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_profile_uri index 65309e588..39a9ef49a 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_profile_uri +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_profile_uri @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_get_endpoints_profile_uri", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_get_endpoints_profile_uri" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.profile_uri_link_id", "target_field": "opcua.profile_uri_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_user_token b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_user_token index 33a3687cc..30d160c9c 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_user_token +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_user_token @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_get_endpoints_user_token", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_get_endpoints_user_token" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.user_token_link_id", "target_field": "opcua.user_token_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_opensecure_channel b/salt/elasticsearch/files/ingest/zeek.opcua_binary_opensecure_channel index 59c41206d..df91874f5 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_opensecure_channel +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_opensecure_channel @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_opensecure_channel", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_opensecure_channel" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcua_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read index 9eee12ff7..07a9134e7 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcua_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims index a0955f534..661a4b49a 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_array_dims", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_array_dims" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.array_dim_link_id", "target_field": "opcua.array_dim_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims_link b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims_link index 94644246f..436aa65d5 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims_link +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims_link @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_array_dims_link", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_array_dims_link" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.variant_data_array_dim_link_id", "target_field": "opcua.variant_data_array_dim_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_diagnostic_info b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_diagnostic_info index 64376bd08..738c39e03 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_diagnostic_info +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_diagnostic_info @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_diagnostic_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_diagnostic_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.read_diag_info_link_id", "target_field": "opcua.read_diag_info_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object index 8ef46251b..b5f76e80b 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_extension_object", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_extension_object" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.ext_obj_link_id", "target_field": "opcua.ext_obj_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object_link b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object_link index 0aae27ca1..463fbdeb5 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object_link +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object_link @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_extension_object_link", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_extension_object_link" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.variant_data_ext_obj_link_id", "target_field": "opcua.variant_data_ext_obj_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_nodes_to_read b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_nodes_to_read index 39c0c25b1..71263a3be 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_nodes_to_read +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_nodes_to_read @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_nodes_to_read", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_nodes_to_read" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.nodes_to_read_link_id", "target_field": "opcua.nodes_to_read_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results index 49d14d404..d049c851b 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_results", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_results" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.results_link_id", "target_field": "opcua.results_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results_link b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results_link index 75245d212..a854b1f2f 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results_link +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results_link @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_results_link", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_results_link" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.read_results_link_id", "target_field": "opcua.read_results_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_status_code b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_status_code index d3b6ece54..5c34f9b61 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_status_code +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_status_code @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_status_code", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_status_code" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.read_status_code_link_id", "target_field": "opcua.read_status_code_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data index d77404bc5..725efaf07 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_variant_data", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_variant_data" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.read_variant_data_link_id", "target_field": "opcua.read_variant_data_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data_link b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data_link index 8585789ff..caad459ed 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data_link +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data_link @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_variant_data_link", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_variant_data_link" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.read_results_variant_data_link_id", "target_field": "opcua.read_results_variant_data_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_status_code_detail b/salt/elasticsearch/files/ingest/zeek.opcua_binary_status_code_detail index e1bff04a4..2a410da2d 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_status_code_detail +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_status_code_detail @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_status_code_detail", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_status_code_detail" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.status_code", "target_field": "opcua.status_code", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.pe b/salt/elasticsearch/files/ingest/zeek.pe index 84f833742..e3bcd678f 100644 --- a/salt/elasticsearch/files/ingest/zeek.pe +++ b/salt/elasticsearch/files/ingest/zeek.pe @@ -1,6 +1,7 @@ { "description" : "zeek.pe", "processors" : [ + { "set": { "field": "event.dataset", "value": "pe" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.id", "target_field": "log.id.fuid", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.profinet b/salt/elasticsearch/files/ingest/zeek.profinet index e9d69c0dc..ca34a305c 100644 --- a/salt/elasticsearch/files/ingest/zeek.profinet +++ b/salt/elasticsearch/files/ingest/zeek.profinet @@ -1,6 +1,7 @@ { "description" : "zeek.profinet", "processors" : [ + { "set": { "field": "event.dataset", "value": "profinet" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true} }, { "rename": { "field": "message2.operation_type", "target_field": "profinet.operation_type", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.profinet_dce_rpc b/salt/elasticsearch/files/ingest/zeek.profinet_dce_rpc index e89fd7d95..c3e200630 100644 --- a/salt/elasticsearch/files/ingest/zeek.profinet_dce_rpc +++ b/salt/elasticsearch/files/ingest/zeek.profinet_dce_rpc @@ -1,6 +1,7 @@ { "description" : "zeek.profinet_dce_rpc", "processors" : [ + { "set": { "field": "event.dataset", "value": "profinet_dce_rpc" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true} }, { "rename": { "field": "message2.version", "target_field": "profinet.version", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.radius b/salt/elasticsearch/files/ingest/zeek.radius index 715f41478..3b25dd32b 100644 --- a/salt/elasticsearch/files/ingest/zeek.radius +++ b/salt/elasticsearch/files/ingest/zeek.radius @@ -1,6 +1,7 @@ { "description" : "zeek.radius", "processors" : [ + { "set": { "field": "event.dataset", "value": "radius" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.username", "target_field": "user.name", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.rdp b/salt/elasticsearch/files/ingest/zeek.rdp index ce792f32a..a5aea03f1 100644 --- a/salt/elasticsearch/files/ingest/zeek.rdp +++ b/salt/elasticsearch/files/ingest/zeek.rdp @@ -1,6 +1,7 @@ { "description" : "zeek.rdp", "processors" : [ + { "set": { "field": "event.dataset", "value": "rdp" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.cookie", "target_field": "rdp.cookie", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.rfb b/salt/elasticsearch/files/ingest/zeek.rfb index fc0653d45..c566827c8 100644 --- a/salt/elasticsearch/files/ingest/zeek.rfb +++ b/salt/elasticsearch/files/ingest/zeek.rfb @@ -1,6 +1,7 @@ { "description" : "zeek.rfb", "processors" : [ + { "set": { "field": "event.dataset", "value": "rfb" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.client_major_version", "target_field": "rfb.client_major_version", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.s7comm b/salt/elasticsearch/files/ingest/zeek.s7comm index e9f5e6318..0af7ee935 100644 --- a/salt/elasticsearch/files/ingest/zeek.s7comm +++ b/salt/elasticsearch/files/ingest/zeek.s7comm @@ -1,6 +1,7 @@ { "description" : "zeek.s7comm", "processors" : [ + { "set": { "field": "event.dataset", "value": "s7comm" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.rosctr_code", "target_field": "s7.ros.control.code", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.s7comm_plus b/salt/elasticsearch/files/ingest/zeek.s7comm_plus index cbb7d5723..e37603752 100644 --- a/salt/elasticsearch/files/ingest/zeek.s7comm_plus +++ b/salt/elasticsearch/files/ingest/zeek.s7comm_plus @@ -1,6 +1,7 @@ { "description" : "zeek.s7comm_plus", "processors" : [ + { "set": { "field": "event.dataset", "value": "s7comm_plus" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.version", "target_field": "s7.version", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.s7comm_read_szl b/salt/elasticsearch/files/ingest/zeek.s7comm_read_szl index c044c08a5..55571584a 100644 --- a/salt/elasticsearch/files/ingest/zeek.s7comm_read_szl +++ b/salt/elasticsearch/files/ingest/zeek.s7comm_read_szl @@ -1,6 +1,7 @@ { "description" : "zeek.s7comm_read_szl", "processors" : [ + { "set": { "field": "event.dataset", "value": "s7comm_read_szl" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.pdu_reference", "target_field": "s7.pdu_reference", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.s7comm_upload_download b/salt/elasticsearch/files/ingest/zeek.s7comm_upload_download index 51d8ca03f..83f773f0a 100644 --- a/salt/elasticsearch/files/ingest/zeek.s7comm_upload_download +++ b/salt/elasticsearch/files/ingest/zeek.s7comm_upload_download @@ -1,6 +1,7 @@ { "description" : "zeek.s7comm_upload_download", "processors" : [ + { "set": { "field": "event.dataset", "value": "s7comm_upload_download" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.rosctr", "target_field": "s7.ros.control.name", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.signatures b/salt/elasticsearch/files/ingest/zeek.signatures index 2a5f23ec7..00381660e 100644 --- a/salt/elasticsearch/files/ingest/zeek.signatures +++ b/salt/elasticsearch/files/ingest/zeek.signatures @@ -1,6 +1,7 @@ { "description" : "zeek.signatures", "processors" : [ + { "set": { "field": "event.dataset", "value": "signatures" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.note", "target_field": "note", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.sip b/salt/elasticsearch/files/ingest/zeek.sip index 5a8627878..a520f5bcb 100644 --- a/salt/elasticsearch/files/ingest/zeek.sip +++ b/salt/elasticsearch/files/ingest/zeek.sip @@ -1,6 +1,7 @@ { "description" : "zeek.sip", "processors" : [ + { "set": { "field": "event.dataset", "value": "sip" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.trans_depth", "target_field": "sip.transaction.depth", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.smb_files b/salt/elasticsearch/files/ingest/zeek.smb_files index da78953fe..192c6f766 100644 --- a/salt/elasticsearch/files/ingest/zeek.smb_files +++ b/salt/elasticsearch/files/ingest/zeek.smb_files @@ -1,6 +1,7 @@ { "description" : "zeek.smb_files", "processors" : [ + { "set": { "field": "event.dataset", "value": "smb_files" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.fuid", "target_field": "log.id.fuid", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.smb_mapping b/salt/elasticsearch/files/ingest/zeek.smb_mapping index 637ff414f..5c6d8bb89 100644 --- a/salt/elasticsearch/files/ingest/zeek.smb_mapping +++ b/salt/elasticsearch/files/ingest/zeek.smb_mapping @@ -1,6 +1,7 @@ { "description" : "zeek.smb_mapping", "processors" : [ + { "set": { "field": "event.dataset", "value": "smb_mapping" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.path", "target_field": "smb.path", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.smtp b/salt/elasticsearch/files/ingest/zeek.smtp index 9bfb1e3e1..e671cce4c 100644 --- a/salt/elasticsearch/files/ingest/zeek.smtp +++ b/salt/elasticsearch/files/ingest/zeek.smtp @@ -1,6 +1,7 @@ { "description" : "zeek.smtp", "processors" : [ + { "set": { "field": "event.dataset", "value": "smtp" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "remove": { "field": "path", "ignore_failure": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.snmp b/salt/elasticsearch/files/ingest/zeek.snmp index 664bc4112..6ee65f776 100644 --- a/salt/elasticsearch/files/ingest/zeek.snmp +++ b/salt/elasticsearch/files/ingest/zeek.snmp @@ -1,6 +1,7 @@ { "description" : "zeek.snmp", "processors" : [ + { "set": { "field": "event.dataset", "value": "snmp" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.duration", "target_field": "event.duration", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.socks b/salt/elasticsearch/files/ingest/zeek.socks index 6ccfa0d1a..24357c85b 100644 --- a/salt/elasticsearch/files/ingest/zeek.socks +++ b/salt/elasticsearch/files/ingest/zeek.socks @@ -1,6 +1,7 @@ { "description" : "zeek.socks", "processors" : [ + { "set": { "field": "event.dataset", "value": "socks" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.version", "target_field": "socks.version", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.software b/salt/elasticsearch/files/ingest/zeek.software index 16a2ae1f1..f5d3d1013 100644 --- a/salt/elasticsearch/files/ingest/zeek.software +++ b/salt/elasticsearch/files/ingest/zeek.software @@ -1,6 +1,7 @@ { "description" : "zeek.software", "processors" : [ + { "set": { "field": "event.dataset", "value": "software" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "dot_expander": { "field": "version.major", "path": "message2", "ignore_failure": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ssh b/salt/elasticsearch/files/ingest/zeek.ssh index 03a9abc15..cffcd854f 100644 --- a/salt/elasticsearch/files/ingest/zeek.ssh +++ b/salt/elasticsearch/files/ingest/zeek.ssh @@ -1,6 +1,7 @@ { "description" : "zeek.ssh", "processors" : [ + { "set": { "field": "event.dataset", "value": "ssh" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.version", "target_field": "ssh.version", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ssl b/salt/elasticsearch/files/ingest/zeek.ssl index e51fd4591..8ae7c8728 100644 --- a/salt/elasticsearch/files/ingest/zeek.ssl +++ b/salt/elasticsearch/files/ingest/zeek.ssl @@ -1,6 +1,7 @@ { "description" : "zeek.ssl", "processors" : [ + { "set": { "field": "event.dataset", "value": "ssl" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.version", "target_field": "ssl.version", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.stun b/salt/elasticsearch/files/ingest/zeek.stun index 48f648d74..322c1030a 100644 --- a/salt/elasticsearch/files/ingest/zeek.stun +++ b/salt/elasticsearch/files/ingest/zeek.stun @@ -1,6 +1,7 @@ { "description" : "zeek.stun", "processors" : [ + { "set": { "field": "event.dataset", "value": "stun" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true} }, { "rename": { "field": "message2.proto", "target_field": "network.protocol", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.stun_nat b/salt/elasticsearch/files/ingest/zeek.stun_nat index 45f9b3055..3ba4f7840 100644 --- a/salt/elasticsearch/files/ingest/zeek.stun_nat +++ b/salt/elasticsearch/files/ingest/zeek.stun_nat @@ -1,6 +1,7 @@ { "description" : "zeek.stun_nat", "processors" : [ + { "set": { "field": "event.dataset", "value": "stun_nat" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true} }, { "rename": { "field": "message2.proto", "target_field": "network.protocol", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.syslog b/salt/elasticsearch/files/ingest/zeek.syslog index 5bdbd9736..4d58430a3 100644 --- a/salt/elasticsearch/files/ingest/zeek.syslog +++ b/salt/elasticsearch/files/ingest/zeek.syslog @@ -1,6 +1,7 @@ { "description" : "zeek.syslog", "processors" : [ + { "set": { "field": "event.dataset", "value": "syslog" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.proto", "target_field": "network.protocol", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.tds b/salt/elasticsearch/files/ingest/zeek.tds index 43c2cad18..9ad7b60bb 100644 --- a/salt/elasticsearch/files/ingest/zeek.tds +++ b/salt/elasticsearch/files/ingest/zeek.tds @@ -1,6 +1,7 @@ { "description" : "zeek.tds", "processors" : [ + { "set": { "field": "event.dataset", "value": "tds" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.command", "target_field": "tds.command", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.tds_rpc b/salt/elasticsearch/files/ingest/zeek.tds_rpc index 75a73c6ba..2915653c3 100644 --- a/salt/elasticsearch/files/ingest/zeek.tds_rpc +++ b/salt/elasticsearch/files/ingest/zeek.tds_rpc @@ -1,6 +1,7 @@ { "description" : "zeek.tds_rpc", "processors" : [ + { "set": { "field": "event.dataset", "value": "tds_rpc" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.procedure_name", "target_field": "tds.procedure_name", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.tds_sql_batch b/salt/elasticsearch/files/ingest/zeek.tds_sql_batch index 560cd1ef3..70f3ef755 100644 --- a/salt/elasticsearch/files/ingest/zeek.tds_sql_batch +++ b/salt/elasticsearch/files/ingest/zeek.tds_sql_batch @@ -1,6 +1,7 @@ { "description" : "zeek.tds_sql_batch", "processors" : [ + { "set": { "field": "event.dataset", "value": "tds_sql_batch" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.header_type", "target_field": "tds.header_type", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.tunnel b/salt/elasticsearch/files/ingest/zeek.tunnel index 66250091e..76b1ca745 100644 --- a/salt/elasticsearch/files/ingest/zeek.tunnel +++ b/salt/elasticsearch/files/ingest/zeek.tunnel @@ -1,6 +1,7 @@ { "description" : "zeek.tunnel", "processors" : [ + { "set": { "field": "event.dataset", "value": "tunnel" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "pipeline": { "name": "zeek.tunnels" } } ] diff --git a/salt/elasticsearch/files/ingest/zeek.tunnels b/salt/elasticsearch/files/ingest/zeek.tunnels index bb4c18268..84fddcda1 100644 --- a/salt/elasticsearch/files/ingest/zeek.tunnels +++ b/salt/elasticsearch/files/ingest/zeek.tunnels @@ -1,6 +1,7 @@ { "description" : "zeek.tunnels", "processors" : [ + { "set": { "field": "event.dataset", "value": "tunnels" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.tunnel_type", "target_field": "tunnel.type", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.weird b/salt/elasticsearch/files/ingest/zeek.weird index 2665bbb33..deb5e8058 100644 --- a/salt/elasticsearch/files/ingest/zeek.weird +++ b/salt/elasticsearch/files/ingest/zeek.weird @@ -1,6 +1,7 @@ { "description" : "zeek.weird", "processors" : [ + { "set": { "field": "event.dataset", "value": "weird" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.name", "target_field": "weird.name", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.wireguard b/salt/elasticsearch/files/ingest/zeek.wireguard index 1df929666..26ee0e0a3 100644 --- a/salt/elasticsearch/files/ingest/zeek.wireguard +++ b/salt/elasticsearch/files/ingest/zeek.wireguard @@ -1,6 +1,7 @@ { "description" : "zeek.wireguard", "processors" : [ + { "set": { "field": "event.dataset", "value": "wireguard" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.established", "target_field": "wireguard.established", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.x509 b/salt/elasticsearch/files/ingest/zeek.x509 index 49a79dbd0..640ea81e3 100644 --- a/salt/elasticsearch/files/ingest/zeek.x509 +++ b/salt/elasticsearch/files/ingest/zeek.x509 @@ -1,6 +1,7 @@ { "description" : "zeek.x509", "processors" : [ + { "set": { "field": "event.dataset", "value": "x509" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.id", "target_field": "log.id.fuid", "ignore_missing": true } }, From 51692ac66c2295029a1cd8894d4d62e81514f31c Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 23 Jan 2023 21:52:44 +0000 Subject: [PATCH 11/72] Update index pattern in various template definitions to match new data stream naming convention --- salt/elasticsearch/defaults.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index ff96d139b..11b445c62 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1165,7 +1165,7 @@ elasticsearch: index_template: data_stream: {} index_patterns: - - logs-*_so + - logs-*-so* template: mappings: dynamic_templates: @@ -1418,7 +1418,7 @@ elasticsearch: index_sorting: False index_template: index_patterns: - - so-elasticsearch* + - logs-elasticsearch-default* template: mappings: dynamic_templates: @@ -2171,7 +2171,7 @@ elasticsearch: index_template: data_stream: {} index_patterns: - - logs-*-suricata_so + - logs-suricata-so* template: mappings: dynamic_templates: @@ -2339,7 +2339,7 @@ elasticsearch: index_template: data_stream: {} index_patterns: - - logs-*-import_so + - logs-import-so* template: mappings: dynamic_templates: @@ -2678,7 +2678,7 @@ elasticsearch: index_sorting: False index_template: index_patterns: - - logs-*-kratos_so + - logs-kratos-so* template: mappings: dynamic_templates: @@ -2755,7 +2755,7 @@ elasticsearch: index_sorting: False index_template: index_patterns: - - so-logstash* + - logs-logstash-default* template: mappings: dynamic_templates: @@ -3676,7 +3676,7 @@ elasticsearch: index_sorting: False index_template: index_patterns: - - so-redis* + - logs-redis-default* template: mappings: dynamic_templates: @@ -4178,7 +4178,7 @@ elasticsearch: index_template: data_stream: {} index_patterns: - - logs-*-strelka_so + - logs-strelka-so* template: mappings: dynamic_templates: @@ -4263,7 +4263,7 @@ elasticsearch: index_sorting: False index_template: index_patterns: - - so-syslog* + - logs-syslog-so* template: mappings: dynamic_templates: @@ -4432,7 +4432,7 @@ elasticsearch: index_template: data_stream: {} index_patterns: - - logs-*-zeek_so + - logs-zeek-so* template: mappings: dynamic_templates: From 88107fe0dfc80e799d2da2d8491268b66ef0b0d6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 24 Jan 2023 08:59:51 -0500 Subject: [PATCH 12/72] remove filebeat and redis(commented out) from telegraf config --- salt/telegraf/etc/telegraf.conf | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index c9cbe70e5..0c3957a89 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -239,19 +239,6 @@ password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" {%- endif %} -{# if grains.role in ['so-eval','so-standalone','so-manager', 'so-managersearch', 'so-heavynode', 'so-receiver'] -%} -[[inputs.redis]] - servers = ["tcp://localhost:6379"] -{%- endif #} - -{%- if grains.role in ['so-searchnode', 'so-sensor', 'so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode'] %} -[[inputs.beat]] - url = "http://127.0.0.1:5066" - include = ["filebeat", "libbeat"] - username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_beats_user:user') }}" - password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_beats_user:pass') }}" -{%- endif %} - # # Read metrics from one or more commands that can output to stdout [[inputs.exec]] commands = [ From ee98e0684eb6887e5d45229e5ce89e64b980bb8c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 24 Jan 2023 09:44:01 -0500 Subject: [PATCH 13/72] change MASTER to MANAGER --- salt/common/tools/sbin/so-minion | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-minion b/salt/common/tools/sbin/so-minion index bde47991f..1f71eb12e 100755 --- a/salt/common/tools/sbin/so-minion +++ b/salt/common/tools/sbin/so-minion @@ -184,11 +184,11 @@ function createHEAVYNODE() { add_sensor_to_minion } -function createMASTER() { +function createMANAGER() { add_elastic_to_minion } -function createMASTERSEARCH() { +function createMANAGERSEARCH() { add_elastic_to_minion } From f19cf753112915aab309e2ada94708f9c203e4fa Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 24 Jan 2023 14:45:00 +0000 Subject: [PATCH 14/72] Change how event.dataset is determined for Suricata events --- salt/elasticsearch/files/ingest/suricata.common | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common index a8be6d46b..6e956d196 100644 --- a/salt/elasticsearch/files/ingest/suricata.common +++ b/salt/elasticsearch/files/ingest/suricata.common @@ -9,12 +9,11 @@ { "rename":{ "field": "message2.dest_ip", "target_field": "destination.ip", "ignore_failure": true } }, { "rename":{ "field": "message2.dest_port", "target_field": "destination.port", "ignore_failure": true } }, { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_missing": true } }, - { "remove":{ "field": "dataset", "ignore_failure": true } }, - { "rename":{ "field": "message2.event_type", "target_field": "dataset", "ignore_failure": true } }, + { "set": { "field": "event.dataset", "value": "{{ message2.event_type }}" } }, { "set": { "field": "observer.name", "value": "{{agent.name}}" } }, { "set": { "field": "event.ingested", "value": "{{@timestamp}}" } }, { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } }, { "remove":{ "field": "agent", "ignore_failure": true } }, - { "pipeline": { "if": "ctx?.dataset != null", "name": "suricata.{{dataset}}" } } + { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } } ] -} \ No newline at end of file +} From 7b4d8a47f0ebbf161e1e260671bb433683a72f14 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 24 Jan 2023 15:07:00 +0000 Subject: [PATCH 15/72] Add copyright header to 'so-elastic-fleet-*' scripts --- salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete | 5 +++++ salt/common/tools/sbin/so-elastic-fleet-agent-policy-list | 5 +++++ salt/common/tools/sbin/so-elastic-fleet-agent-policy-view | 5 +++++ salt/common/tools/sbin/so-elastic-fleet-data-streams-list | 5 +++++ .../sbin/so-elastic-fleet-integration-policy-bulk-delete | 5 +++++ .../tools/sbin/so-elastic-fleet-integration-policy-delete | 5 +++++ .../tools/sbin/so-elastic-fleet-integration-policy-list | 5 +++++ .../tools/sbin/so-elastic-fleet-integration-policy-load | 4 ++++ 8 files changed, 39 insertions(+) diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete index 0564ad3c6..fe21daae0 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete +++ b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete @@ -1,4 +1,9 @@ #/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + . /usr/sbin/so-common POLICY_ID=$1 diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list index 8a9638922..479fdc72d 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list +++ b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list @@ -1,4 +1,9 @@ #/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + . /usr/sbin/so-common # Let's snag a cookie from Kibana diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view index 6ab6474b0..5f267220b 100644 --- a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view +++ b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view @@ -1,4 +1,9 @@ #/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + . /usr/sbin/so-common POLICY_ID=$1 diff --git a/salt/common/tools/sbin/so-elastic-fleet-data-streams-list b/salt/common/tools/sbin/so-elastic-fleet-data-streams-list index 9348428e8..238fc8ada 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-data-streams-list +++ b/salt/common/tools/sbin/so-elastic-fleet-data-streams-list @@ -1,4 +1,9 @@ #/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + . /usr/sbin/so-common # Let's snag a cookie from Kibana diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete index e2c3095d6..9c2061cd9 100644 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete @@ -1,4 +1,9 @@ #/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + . /usr/sbin/so-common POLICY_ID=$1 diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete index b66b7eca8..a8601028c 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete @@ -1,4 +1,9 @@ #/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + . /usr/sbin/so-common POLICY_ID=$1 diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list index c73e84174..07d2822fb 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list @@ -1,4 +1,9 @@ #/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + . /usr/sbin/so-common # Let's snag a cookie from Kibana diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index 1a5523723..f4924aac4 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -1,4 +1,8 @@ #/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. . /usr/sbin/so-common From 2da30f42d414cb7c03b683a68f2f2221b52b8247 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 24 Jan 2023 10:07:32 -0500 Subject: [PATCH 16/72] Check for Ubuntu --- setup/so-functions | 11 +++++++++++ setup/so-setup | 10 ++++++++++ setup/so-whiptail | 22 ++++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 0005e90b4..50e1704d9 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2303,6 +2303,17 @@ so_add_user() { fi } +ubuntu_check() { + if [[ $OS == "ubuntu" ]]; then + if [[ $waitforstate ]]; then + whiptail_ubuntu_notsupported + exit 1 + else + whiptail_ubuntu_warning + fi + fi +} + update_sudoers_for_testing() { if [ -n "$TESTING" ]; then info "Ensuring $INSTALLUSERNAME has password-less sudo access for automated testing purposes." diff --git a/setup/so-setup b/setup/so-setup index 2b0abedc8..bbc3ae971 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -314,6 +314,7 @@ if ! [[ -f $install_opt_file ]]; then # If you are a manager ask ALL the manager things here. I know there is code re-use but this makes it easier to add new roles. if [[ $is_eval ]]; then waitforstate=true + ubuntu_check monints=true check_elastic_license check_requirements "manager" @@ -336,6 +337,7 @@ if ! [[ -f $install_opt_file ]]; then whiptail_end_settings elif [[ $is_standalone ]]; then waitforstate=true + ubuntu_check monints=true check_elastic_license check_requirements "manager" @@ -359,6 +361,7 @@ if ! [[ -f $install_opt_file ]]; then elif [[ $is_manager ]]; then check_elastic_license waitforstate=true + ubuntu_check check_requirements "manager" networking_needful collect_dockernet @@ -378,6 +381,7 @@ if ! [[ -f $install_opt_file ]]; then elif [[ $is_managersearch ]]; then check_elastic_license waitforstate=true + ubuntu_check check_requirements "manager" networking_needful collect_dockernet @@ -395,6 +399,7 @@ if ! [[ -f $install_opt_file ]]; then collect_so_allow whiptail_end_settings elif [[ $is_sensor ]]; then + ubuntu_check monints=true check_requirements "sensor" calculate_useable_cores @@ -410,6 +415,7 @@ if ! [[ -f $install_opt_file ]]; then whiptail_end_settings elif [[ $is_searchnode ]]; then + ubuntu_check check_requirements "elasticsearch" networking_needful check_network_manager_conf @@ -422,6 +428,7 @@ if ! [[ -f $install_opt_file ]]; then whiptail_end_settings elif [[ $is_heavynode ]]; then + ubuntu_check monints=true check_requirements "heavynode" calculate_useable_cores @@ -432,6 +439,7 @@ if ! [[ -f $install_opt_file ]]; then whiptail_end_settings elif [[ $is_idh ]]; then + ubuntu_check check_requirements "idh" networking_needful collect_mngr_hostname @@ -440,6 +448,7 @@ if ! [[ -f $install_opt_file ]]; then whiptail_end_settings elif [[ $is_import ]]; then + ubuntu_check waitforstate=true monints=true check_elastic_license @@ -461,6 +470,7 @@ if ! [[ -f $install_opt_file ]]; then whiptail_end_settings elif [[ $is_receiver ]]; then + ubuntu_check check_requirements "receiver" networking_needful collect_mngr_hostname diff --git a/setup/so-whiptail b/setup/so-whiptail index a03c85645..5615edf41 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1327,6 +1327,28 @@ whiptail_storage_requirements() { whiptail_check_exitstatus $exitstatus } +whiptail_ubuntu_notsupported() { + [ -n "$TESTING" ] && return + + read -r -d '' message <<- EOM + Ubuntu is not supported for this node type. + + Please use a supported OS or install via ISO. + EOM + whiptail --title "$whiptail_title" --msgbox "$message" 14 75 +} + +whiptail_ubuntu_warning() { + [ -n "$TESTING" ] && return + + read -r -d '' message <<- EOM + Ubuntu support for this node type is limited. + + Please consider using a fully supported OS or install via ISO. + EOM + whiptail --title "$whiptail_title" --msgbox "$message" 14 75 + +} whiptail_uppercase_warning() { [ -n "$TESTING" ] && return From 50895ee30475fadcd09d5a36cd508590f6620826 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 24 Jan 2023 10:16:58 -0500 Subject: [PATCH 17/72] need to set_minion_info in setup for each node type --- setup/so-setup | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup/so-setup b/setup/so-setup index 5088395f3..2644ca220 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -366,6 +366,7 @@ if ! [[ -f $install_opt_file ]]; then whiptail_airgap fi detect_cloud + set_minion_info set_default_log_size >> $setup_log 2>&1 info "Verifying all network devices are managed by Network Manager that should be" check_network_manager_conf @@ -385,6 +386,7 @@ if ! [[ -f $install_opt_file ]]; then whiptail_airgap fi detect_cloud + set_minion_info set_default_log_size >> $setup_log 2>&1 info "Verifying all network devices are managed by Network Manager that should be" check_network_manager_conf @@ -429,6 +431,7 @@ if ! [[ -f $install_opt_file ]]; then collect_mngr_hostname add_mngr_ip_to_hosts check_manager_connection + set_minion_info whiptail_end_settings elif [[ $is_idh ]]; then @@ -437,6 +440,7 @@ if ! [[ -f $install_opt_file ]]; then collect_mngr_hostname add_mngr_ip_to_hosts check_manager_connection + set_minion_info whiptail_end_settings elif [[ $is_import ]]; then @@ -466,6 +470,7 @@ if ! [[ -f $install_opt_file ]]; then collect_mngr_hostname add_mngr_ip_to_hosts check_manager_connection + set_minion_info whiptail_end_settings fi From cfc232eafa4d7c480f2373c0750bdc1eae75d845 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 24 Jan 2023 10:23:42 -0500 Subject: [PATCH 18/72] try paths filter on both even though docs only mention support for push --- .github/workflows/pythontest.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 241245044..f888b1a43 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -1,6 +1,10 @@ name: python-test -on: [push, pull_request] +on: + push: + paths: salt/sensoroni/files/analyzers + pull_request: + paths: salt/sensoroni/files/analyzers jobs: build: From ec2e923530419d43d59895d69b0ab412f7e9c78a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 24 Jan 2023 10:28:39 -0500 Subject: [PATCH 19/72] Add proper spacing between headers and content --- salt/sensoroni/files/analyzers/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/sensoroni/files/analyzers/README.md b/salt/sensoroni/files/analyzers/README.md index e87a95638..8b1f44f29 100644 --- a/salt/sensoroni/files/analyzers/README.md +++ b/salt/sensoroni/files/analyzers/README.md @@ -3,6 +3,7 @@ Security Onion provides a means for performing data analysis on varying inputs. This data can be any data of interest sourced from event logs. Examples include hostnames, IP addresses, file hashes, URLs, etc. The analysis is conducted by one or more analyzers that understand that type of input. Analyzers come with the default installation of Security Onion. However, it is also possible to add additional analyzers to extend the analysis across additional areas or data types. ## Supported Observable Types + The built-in analyzers support the following observable types: | Name | Domain | Hash | IP | Mail | Other | URI | URL | User Agent | @@ -20,6 +21,7 @@ The built-in analyzers support the following observable types: | WhoisLookup |✓ |✗|✗|✗|✗|✓|✗|✗| ## Authentication + Many analyzers require authentication, via an API key or similar. The table below illustrates which analyzers require authentication. | Name | Authn Req'd| From 62b96c3698907dec2feba42cc56d4990cd0f3a50 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 24 Jan 2023 10:31:02 -0500 Subject: [PATCH 20/72] rework filter for action --- .github/workflows/pythontest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index f888b1a43..dae1c420b 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -2,9 +2,9 @@ name: python-test on: push: - paths: salt/sensoroni/files/analyzers + paths: + - "salt/sensoroni/files/analyzers/**" pull_request: - paths: salt/sensoroni/files/analyzers jobs: build: From 0dc5e7e714caf1ed3e4b242f9c79625b6e254cba Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 24 Jan 2023 10:38:59 -0500 Subject: [PATCH 21/72] try paths with wildcard --- .github/workflows/pythontest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index dae1c420b..3ce481ee0 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -5,6 +5,8 @@ on: paths: - "salt/sensoroni/files/analyzers/**" pull_request: + paths: + - "salt/sensoroni/files/analyzers/**" jobs: build: From b0709e93fa1aeaf9d12907d2270dd77481040a37 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 24 Jan 2023 10:50:52 -0500 Subject: [PATCH 22/72] test workflow --- salt/soc/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 40cb4487d..a2ecc610e 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -115,7 +115,7 @@ so-soc: {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - {{ SN.split('_')|first }}:{{ SNDATA.ip }} {%- endfor %} - {%- endif %} + {%- endif %} - port_bindings: {% for BINDING in DOCKER.containers['so-soc'].port_bindings %} - {{ BINDING }} From b23575d85e36cbce492c28f0494f9d95307652ee Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 24 Jan 2023 11:03:03 -0500 Subject: [PATCH 23/72] add global vars for manager --- salt/vars/manager.map.jinja | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 salt/vars/manager.map.jinja diff --git a/salt/vars/manager.map.jinja b/salt/vars/manager.map.jinja new file mode 100644 index 000000000..c6b348341 --- /dev/null +++ b/salt/vars/manager.map.jinja @@ -0,0 +1,15 @@ +{% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %} +{% from 'vars/logstash.map.jinja' import LOGSTASH_GLOBALS %} + +{% set ROLE_GLOBALS = {} %} + +{% set MANAGER_GLOBALS = + [ + ELASTICSEARCH_GLOBALS, + LOGSTASH_GLOBALS + ] +%} + +{% for sg in MANAGER_GLOBALS %} +{% do salt['defaults.merge'](ROLE_GLOBALS, sg, merge_lists=False, in_place=True) %} +{% endfor %} From 1e5377c78ab26e366cf39280ab0b91665d16fc25 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 24 Jan 2023 16:56:20 +0000 Subject: [PATCH 24/72] Condense RITA integration policies, add ICS tags, and improve output readability --- ...astic-fleet-integration-policy-bulk-delete | 2 + .../so-elastic-fleet-integration-policy-load | 48 +++++++++++++------ 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete index 9c2061cd9..4596774b4 100644 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete @@ -18,4 +18,6 @@ for i in $INTEGRATION_POLICY_IDS; do # Delete integration policies echo "Deleting integration policy: $i..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d"{\"packagePolicyIds\": [\"$i\"], \"force\":true}"; + echo + echo done diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index f4924aac4..26d3c73e1 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -21,58 +21,76 @@ echo echo "Setting up default Security Onion package policies for Elastic Agent..." # Set up Suricata logs +echo echo "Setting up Suricata package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "suricata-logs", "name": "suricata-logs", "description": "Suricata integration", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/suricata/eve*.json" ], "data_stream.dataset": "suricata", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata", "custom": "pipeline: suricata.common" }}}}}}' +echo # Set up Zeek logs +echo echo "Setting up Zeek package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "zeek-logs", "description": "Zeek logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/zeek/logs/current/*.log"], "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek", "custom": "prospector.scanner.exclude_files: [\"(broker | capture_loss | loaded_scripts | packet_filter | stats | stderr | stdout).log$\"]\n" } } } } } }' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "zeek-logs", "description": "Zeek logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/zeek/logs/current/*.log"], "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", "custom": "prospector.scanner.exclude_files: [\"(broker | capture_loss | loaded_scripts | packet_filter | stats | stderr | stdout).log$\"]\n" } } } } } }' +echo # Import - Suricata +echo echo "Settings up Suricata import package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-suricata-logs", "name": "import-suricata-logs", "description": "Import Suricata logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/suricata/eve*.json"], "data_stream.dataset": "import", "tags": ["import"], "processors": "- add_fields:\n target: event\n fields:\n category: file\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"", "custom": "pipeline: suricata.common" } } } } } }' +echo # Set Import - Zeek logs +echo echo "Setting up Zeek Import package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "import-zeek-logs", "description": "Zeek Import logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/zeek/logs/*.log"], "data_stream.dataset": "import", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n imported: true", "custom": "prospector.scanner.exclude_files: [\"(broker | capture_loss | loaded_scripts | packet_filter | stats | stderr | stdout).log$\"]" } } } } } }' - +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "import-zeek-logs", "description": "Zeek Import logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/zeek/logs/*.log"], "data_stream.dataset": "import", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"import.file\").slice(0,-4);\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n imported: true\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"" } } } } } }' # Strelka logs echo "Setting up Strelka package policy" curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "strelka-logs", "name": "strelka-logs", "description": "Strelka logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], "data_stream.dataset": "file", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: file\n module: strelka", "custom": "pipeline: strelka.file" }}}}}}' +echo # Syslog TCP Port 514 +echo echo "Setting up Syslog TCP package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "tcp", "version": "1.5.0" }, "id": "syslog-tcp-514", "name": "syslog-tcp-514", "description": "Syslog Over TCP Port 514", "namespace": "so", "inputs": { "tcp-tcp": { "enabled": true, "streams": { "tcp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "processors": "- add_fields:\n target: event\n fields:\n module: syslog", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' +echo # Syslog UDP Port 514 +echo echo "Setting up Syslog UDP package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "udp", "version": "1.5.0" }, "id": "syslog-udp-514", "name": "syslog-udp-514", "description": "Syslog over UDP Port 514", "namespace": "so", "inputs": { "udp-udp": { "enabled": true, "streams": { "udp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "max_message_size": "10KiB", "keep_null": false, "processors": "- add_fields:\n target: event\n fields: \n module: syslog\n", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' +echo # Kratos logs +echo echo "Setting up Kratos package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "access", "tags": [],"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- rename:\n fields:\n - from: \"audience\"\n to: \"event.dataset\"\n ignore_missing: true\n- add_fields:\n when:\n not: \n has_fields: ['event.dataset']\n target: ''\n fields:\n event.dataset: access", "custom": "pipeline: kratos" }}}}}}' +echo -# RITA Beacon logs -echo "Setting up RITA Beacon package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-beacon-logs", "name": "rita-beacon-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv" ], "data_stream.dataset": "beacon", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.beacon" }}}}}}' - -# RITA Connection Logs -echo "Setting up RITA connection package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-connection-logs", "name": "rita-connection-logs", "description": "RITA Connection logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/long-connections.csv" ], "data_stream.dataset": "connections", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.connections" }}}}}}' - -# RITA DNS Logs -echo "Setting up RITA DNS package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-exploded-dns-logs", "name": "rita-exploded-dns-logs", "description": "RITA Exploded DNS logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "exploded_dns", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.dns" }}}}}}' +# RITA Logs +echo +echo "Setting up RITA package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita\n- if:\n log.file.path: beacons.csv\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.beacon\n- if:\n regexp:\n log.file.path: \"*connections.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.connection\n- if:\n log.file.path: \"exploded-dns.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.dns" }}}}}}' +echo # Elasticsearch logs +echo +echo "Seting up Elasticsearch package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "elasticsearch", "version": "1.0.0" }, "id": "elasticsearch-logs", "name": "elasticsearch-logs", "description": "Elasticsearch Logs", "namespace": "default", "inputs": { "elasticsearch-logfile": { "enabled": true, "streams": { "elasticsearch.audit": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_audit.json" ] } }, "elasticsearch.deprecation": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_deprecation.json" ] } }, "elasticsearch.gc": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/gc.log.[0-9]*", "/var/log/elasticsearch/gc.log" ] } }, "elasticsearch.server": { "enabled": true, "vars": { "paths": [ "/opt/so/log/elasticsearch/*.log" ] } }, "elasticsearch.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_index_search_slowlog.json", "/var/log/elasticsearch/*_index_indexing_slowlog.json" ] } } } }, "elasticsearch-elasticsearch/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9200" ], "scope": "node" }, "streams": { "elasticsearch.stack_monitoring.ccr": { "enabled": false }, "elasticsearch.stack_monitoring.cluster_stats": { "enabled": false }, "elasticsearch.stack_monitoring.enrich": { "enabled": false }, "elasticsearch.stack_monitoring.index": { "enabled": false }, "elasticsearch.stack_monitoring.index_recovery": { "enabled": false, "vars": { "active.only": true } }, "elasticsearch.stack_monitoring.index_summary": { "enabled": false }, "elasticsearch.stack_monitoring.ml_job": { "enabled": false }, "elasticsearch.stack_monitoring.node": { "enabled": false }, "elasticsearch.stack_monitoring.node_stats": { "enabled": false }, "elasticsearch.stack_monitoring.pending_tasks": { "enabled": false }, "elasticsearch.stack_monitoring.shard": { "enabled": false } } } } }' +echo # Logstash logs +echo +echo "Setting up Logstash package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "logstash", "version": "2.0.0" }, "id": "logstash-logs", "name": "logstash-logs", "description": "Logstash logs", "namespace": "default", "inputs": { "logstash-logfile": { "enabled": true, "streams": { "logstash.log": { "enabled": true, "vars": { "paths": [ "/opt/so/logs/logstash/logstash.log" ] } }, "logstash.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/logstash/logstash-slowlog-plain*.log", "/var/log/logstash/logstash-slowlog-json*.log" ] } } } }, "logstash-logstash/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9600" ], "period": "10s" }, "streams": { "logstash.stack_monitoring.node": { "enabled": false }, "logstash.stack_monitoring.node_stats": { "enabled": false } } } } }' +echo # Kibana logs +echo +echo "Setting up Kibana package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "kibana", "version": "2.0.0" }, "id": "kibana-logs", "name": "kibana-logs", "description": "Kibana logs", "namespace": "default", "inputs": { "kibana-logfile": { "enabled": true, "streams": { "kibana.audit": { "enabled": false, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } }, "kibana.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } } } }, "kibana-kibana/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:5601" ] }, "streams": { "kibana.stack_monitoring.cluster_actions": { "enabled": false }, "kibana.stack_monitoring.cluster_rules": { "enabled": false }, "kibana.stack_monitoring.node_actions": { "enabled": false }, "kibana.stack_monitoring.node_rules": { "enabled": false }, "kibana.stack_monitoring.stats": { "enabled": false }, "kibana.stack_monitoring.status": { "enabled": false } } } } }' +echo # Redis logs +echo +echo "Setting up Redis package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "redis", "version": "1.4.0" }, "id": "redis-logs", "name": "redis-logs", "description": "Redis logs", "namespace": "default", "inputs": { "redis-logfile": { "enabled": true, "streams": { "redis.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/redis/redis.log" ], "tags": [ "redis-log" ], "preserve_original_event": false } } } }, "redis-redis": { "enabled": false, "streams": { "redis.slowlog": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "password": "" } } } }, "redis-redis/metrics": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "idle_timeout": "20s", "maxconn": 10, "network": "tcp", "password": "" }, "streams": { "redis.info": { "enabled": false, "vars": { "period": "10s" } }, "redis.key": { "enabled": false, "vars": { "key.patterns": "- limit: 20\n pattern: '*'\n", "period": "10s" } }, "redis.keyspace": { "enabled": false, "vars": { "period": "10s" } } } } } }' - +echo From 44d149b1c3c1ebb6018306ca80ee814ac1ce9dc1 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 24 Jan 2023 17:01:52 +0000 Subject: [PATCH 25/72] Allow imported data to use a tag of 'import' --- .../pipelines/config/so/9805_output_elastic_agent.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja index 1c770c7b0..0a148155c 100644 --- a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja +++ b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja @@ -1,5 +1,5 @@ output { - if "elastic-agent" in [tags] and "import" not in [tags] { + if "elastic-agent" in [tags] { if [metadata][pipeline] { elasticsearch { hosts => "{{ GLOBALS.manager }}" From 38ead7cb8264bd3b30b31490c28b01c8fa12151c Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 24 Jan 2023 17:58:19 +0000 Subject: [PATCH 26/72] Remove import tag for now --- salt/common/tools/sbin/so-elastic-fleet-integration-policy-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index 26d3c73e1..aa31e9fb6 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -35,7 +35,7 @@ echo # Import - Suricata echo echo "Settings up Suricata import package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-suricata-logs", "name": "import-suricata-logs", "description": "Import Suricata logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/suricata/eve*.json"], "data_stream.dataset": "import", "tags": ["import"], "processors": "- add_fields:\n target: event\n fields:\n category: file\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"", "custom": "pipeline: suricata.common" } } } } } }' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-suricata-logs", "name": "import-suricata-logs", "description": "Import Suricata logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/suricata/eve*.json"], "data_stream.dataset": "import", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: file\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"", "custom": "pipeline: suricata.common" } } } } } }' echo # Set Import - Zeek logs From 4b9c92c53d91f0dd2639d68228b028e18dd95ce0 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 24 Jan 2023 18:00:34 +0000 Subject: [PATCH 27/72] Set RITA event.dataset value explicitly --- salt/elasticsearch/files/ingest/rita.beacon | 4 ++-- salt/elasticsearch/files/ingest/rita.connection | 4 ++-- salt/elasticsearch/files/ingest/rita.dns | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/elasticsearch/files/ingest/rita.beacon b/salt/elasticsearch/files/ingest/rita.beacon index 2c6138262..81bd34e9d 100644 --- a/salt/elasticsearch/files/ingest/rita.beacon +++ b/salt/elasticsearch/files/ingest/rita.beacon @@ -3,8 +3,8 @@ "processors": [ { "set": { - "field": "_index", - "value": "so-rita", + "field": "event.dataset", + "value": "beacon", "override": true } }, diff --git a/salt/elasticsearch/files/ingest/rita.connection b/salt/elasticsearch/files/ingest/rita.connection index 58cc921f4..2c0e33c87 100644 --- a/salt/elasticsearch/files/ingest/rita.connection +++ b/salt/elasticsearch/files/ingest/rita.connection @@ -3,8 +3,8 @@ "processors": [ { "set": { - "field": "_index", - "value": "so-rita", + "field": "event.dataset", + "value": "connection", "override": true } }, diff --git a/salt/elasticsearch/files/ingest/rita.dns b/salt/elasticsearch/files/ingest/rita.dns index 7583bc320..837571b70 100644 --- a/salt/elasticsearch/files/ingest/rita.dns +++ b/salt/elasticsearch/files/ingest/rita.dns @@ -3,8 +3,8 @@ "processors": [ { "set": { - "field": "_index", - "value": "so-rita", + "field": "event.dataset", + "value": "dns", "override": true } }, From 7b1f867ac3941e20bd471e43abec75e2e4704c18 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 24 Jan 2023 13:17:50 -0500 Subject: [PATCH 28/72] Add defaults for auto extracted observables --- salt/soc/defaults.yaml | 3 +++ salt/soc/soc_soc.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 857f245d1..cec11273b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1020,6 +1020,9 @@ soc: cacheMs: 300000 verifyCert: false casesEnabled: true + extractCommonObservables: + - source.ip + - destination.ip timeoutMs: 300000 timeShiftMs: 120000 defaultDurationMs: 1800000 diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 7d96ca46b..e6b43cf0b 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -69,6 +69,9 @@ soc: description: Set to true if the SOC case management module, natively integrated with Elasticsearch, should be enabled. global: True advanced: True + extractCommonObservables: + description: List of indexed fields to automatically extract into a case observable, when attaching related events to a case. + global: True timeShiftMs: description: Duration (in milliseconds) to further expand the PCAP time range when querying PCAP data related to an event. This duration is added to the normal duration value (see defaultDurationMs). global: True From a048034f168f7818f3d08bf8e30f9d2dcd24b8b9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 24 Jan 2023 13:38:39 -0500 Subject: [PATCH 29/72] Salt for Ubuntu --- setup/so-functions | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 50e1704d9..53d40be4b 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1990,6 +1990,42 @@ repo_sync_local() { } saltify() { + if [[ $is_ubuntu ]]; then + + DEBIAN_FRONTEND=noninteractive retry 150 20 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || exit 1 + update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10 + local pkg_arr=( + 'apache2-utils' + 'ca-certificates' + 'curl' + 'software-properties-common' + 'apt-transport-https' + 'openssl' + 'netcat' + 'jq' + ) + retry 150 20 "apt-get -y install ${pkg_arr[*]}" || exit 1 + + # Add SO Saltstack Repo + logCmd "wget -q --inet4-only -O - https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt/SALTSTACK-GPG-KEY.pub | apt-key add -" + echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt3004.2/ focal main" > /etc/apt/sources.list.d/saltstack.list + + # Add Docker Repo + logCmd "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -" + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + # Ain't nothing but a GPG + mkdir -p "$temp_install_dir"/gpg + apt-key add "$temp_install_dir"/gpg/SALTSTACK-GPG-KEY.pub + apt-key add "$temp_install_dir"/gpg/GPG-KEY-WAZUH + + retry 150 20 "apt-get update" "" "Err:" || exit 1 + retry 150 20 "apt-get -y install salt-minion=3004.2+ds-1 salt-common=3004.2+ds-1" || exit 1 + retry 150 20 "apt-mark hold salt-minion salt-common" || exit 1 + retry 150 20 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb python3-lxml" || exit 1 + + fi + if [[ $is_centos ]]; then RUSALTY=$(rpm -qa | grep salt-minion | wc -l) if [[ "$RUSALTY" -gt 0 ]]; then From b89e7efeea39310ab1807515912b065217548b27 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 24 Jan 2023 15:30:46 -0500 Subject: [PATCH 30/72] Salt for Ubuntu --- setup/so-functions | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-functions b/setup/so-functions index 53d40be4b..0f06f7861 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -911,6 +911,7 @@ detect_os() { OS=ubuntu if grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then OSVER=focal + is_ubuntu=true else info "We do not support your current version of Ubuntu." exit 1 From 20f7a77886abcc235833c8295eaa42ae89a0cd37 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 24 Jan 2023 15:43:12 -0500 Subject: [PATCH 31/72] Salt for Ubuntu --- setup/so-functions | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 0f06f7861..3299509e1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2007,18 +2007,20 @@ saltify() { ) retry 150 20 "apt-get -y install ${pkg_arr[*]}" || exit 1 + logCmd "mkdir -vp /opt/so/gpg" + logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.securityonion.net/file/securityonion-repo/ubuntu/$ubuntu_version/amd64/salt/SALTSTACK-GPG-KEY.pub" + logCmd "wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg" + # Add SO Saltstack Repo logCmd "wget -q --inet4-only -O - https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt/SALTSTACK-GPG-KEY.pub | apt-key add -" - echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt3004.2/ focal main" > /etc/apt/sources.list.d/saltstack.list + echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt3004.2/focal main" > /etc/apt/sources.list.d/saltstack.list + curl -fsSL -o /usr/share/keyrings/salt-archive-keyring.gpg https://repo.saltproject.io/py3/ubuntu/20.04/amd64/3004/salt-archive-keyring.gpg # Add Docker Repo logCmd "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -" add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" # Ain't nothing but a GPG - mkdir -p "$temp_install_dir"/gpg - apt-key add "$temp_install_dir"/gpg/SALTSTACK-GPG-KEY.pub - apt-key add "$temp_install_dir"/gpg/GPG-KEY-WAZUH retry 150 20 "apt-get update" "" "Err:" || exit 1 retry 150 20 "apt-get -y install salt-minion=3004.2+ds-1 salt-common=3004.2+ds-1" || exit 1 From 177ddc11833093fc01b8869cf87402e1231321ac Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 24 Jan 2023 15:48:48 -0500 Subject: [PATCH 32/72] Salt for Ubuntu --- setup/so-functions | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 3299509e1..9629c0200 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2014,7 +2014,6 @@ saltify() { # Add SO Saltstack Repo logCmd "wget -q --inet4-only -O - https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt/SALTSTACK-GPG-KEY.pub | apt-key add -" echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt3004.2/focal main" > /etc/apt/sources.list.d/saltstack.list - curl -fsSL -o /usr/share/keyrings/salt-archive-keyring.gpg https://repo.saltproject.io/py3/ubuntu/20.04/amd64/3004/salt-archive-keyring.gpg # Add Docker Repo logCmd "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -" From 5a223981ca1dbb034ac0b84ec9734c953c59efd7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 24 Jan 2023 15:57:05 -0500 Subject: [PATCH 33/72] Salt for Ubuntu --- setup/so-functions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 9629c0200..6f7a89313 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2010,13 +2010,13 @@ saltify() { logCmd "mkdir -vp /opt/so/gpg" logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.securityonion.net/file/securityonion-repo/ubuntu/$ubuntu_version/amd64/salt/SALTSTACK-GPG-KEY.pub" logCmd "wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg" - + logCmd "apt-key add /opt/so/gpg/SALTSTACK-GPG-KEY.pub" + logCmd "apt-key add /opt/so/gpg/docker.pub" + # Add SO Saltstack Repo - logCmd "wget -q --inet4-only -O - https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt/SALTSTACK-GPG-KEY.pub | apt-key add -" echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt3004.2/focal main" > /etc/apt/sources.list.d/saltstack.list # Add Docker Repo - logCmd "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -" add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" # Ain't nothing but a GPG From 53d6823ba7f6a3b81fc1ba488199da07706639a4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 24 Jan 2023 16:00:03 -0500 Subject: [PATCH 34/72] Salt for Ubuntu --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 6f7a89313..41e877ab8 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2008,11 +2008,11 @@ saltify() { retry 150 20 "apt-get -y install ${pkg_arr[*]}" || exit 1 logCmd "mkdir -vp /opt/so/gpg" - logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.securityonion.net/file/securityonion-repo/ubuntu/$ubuntu_version/amd64/salt/SALTSTACK-GPG-KEY.pub" + logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt/SALTSTACK-GPG-KEY.pub" logCmd "wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg" logCmd "apt-key add /opt/so/gpg/SALTSTACK-GPG-KEY.pub" logCmd "apt-key add /opt/so/gpg/docker.pub" - + # Add SO Saltstack Repo echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt3004.2/focal main" > /etc/apt/sources.list.d/saltstack.list From d5f8ea866101f9e2bbcb0190e6995d0c0c7707a3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 24 Jan 2023 16:05:16 -0500 Subject: [PATCH 35/72] Salt for Ubuntu --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 41e877ab8..cff943376 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2014,7 +2014,7 @@ saltify() { logCmd "apt-key add /opt/so/gpg/docker.pub" # Add SO Saltstack Repo - echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt3004.2/focal main" > /etc/apt/sources.list.d/saltstack.list + echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt3004.2/ focal main" > /etc/apt/sources.list.d/saltstack.list # Add Docker Repo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" From 161881efbbeaefcdfd74926bbc44ccae0f4fec30 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 24 Jan 2023 16:25:26 -0500 Subject: [PATCH 36/72] Salt for Ubuntu --- salt/repo/client/ubuntu.sls | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 salt/repo/client/ubuntu.sls diff --git a/salt/repo/client/ubuntu.sls b/salt/repo/client/ubuntu.sls new file mode 100644 index 000000000..a5d55fa5e --- /dev/null +++ b/salt/repo/client/ubuntu.sls @@ -0,0 +1,20 @@ +# this removes the repo file left by bootstrap-salt.sh without -r +remove_salt.list: + file.absent: + - name: /etc/apt/sources.list.d/salt.list + +saltstack.list: + file.managed: + - name: /etc/apt/sources.list.d/saltstack.list + - contents: + - deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/{{grains.osrelease}}/amd64/salt3004.2/ {{grains.oscodename}} main + +apt_update: + cmd.run: + - name: apt-get update + - onchanges: + - file: saltstack.list + - timeout: 30 + - retry: + attempts: 5 + interval: 30 \ No newline at end of file From d1460ae01fd5eed2c2418f47c89298faba5c65fa Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 24 Jan 2023 17:05:40 -0500 Subject: [PATCH 37/72] add node_data.ips pillar. grab influx host ip for soc extra_hosts --- pillar/node_data/ips.sls | 10 ++++------ pillar/top.sls | 1 + salt/soc/init.sls | 3 ++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pillar/node_data/ips.sls b/pillar/node_data/ips.sls index 233038e66..59c598879 100644 --- a/pillar/node_data/ips.sls +++ b/pillar/node_data/ips.sls @@ -1,7 +1,5 @@ {% set node_types = {} %} {% set manage_alived = salt.saltutil.runner('manage.alived', show_ip=True) %} -{% set manager = grains.master %} -{% set manager_type = manager.split('_')|last %} {% for minionid, ip in salt.saltutil.runner('mine.get', tgt='*', fun='network.ip_addrs', tgt_type='glob') | dictsort() %} {% set hostname = minionid.split('_')[0] %} {% set node_type = minionid.split('_')[1] %} @@ -24,10 +22,10 @@ node_data: {% for node_type, host_values in node_types.items() %} - {{node_type}}: {% for hostname, details in host_values.items() %} - {{hostname}}: - ip: {{details.ip}} - alive: {{ details.alive }} + {{hostname}}: + ip: {{details.ip}} + alive: {{ details.alive }} + role: {{node_type}} {% endfor %} {% endfor %} diff --git a/pillar/top.sls b/pillar/top.sls index 074a0a9d4..d24b6980f 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -10,6 +10,7 @@ base: - sensoroni.adv_sensoroni - telegraf.soc_telegraf - telegraf.adv_telegraf + - node_data.ips '* and not *_eval and not *_import': - logstash.nodes diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 40cb4487d..65cae16e6 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -110,8 +110,9 @@ so-soc: - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw - /opt/so/conf/soc/salt:/opt/sensoroni/salt:rw - /opt/so/saltstack:/opt/so/saltstack:rw - {%- if salt['pillar.get']('nodestab', {}) %} - extra_hosts: + - {{GLOBALS.influxdb_host}}:{{pillar.node_data[GLOBALS.influxdb_host].ip}} + {%- if salt['pillar.get']('nodestab', {}) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - {{ SN.split('_')|first }}:{{ SNDATA.ip }} {%- endfor %} From 7bf9d77962768af01e981ec491e3f4e9fbe318df Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 25 Jan 2023 08:18:21 -0500 Subject: [PATCH 38/72] Rename Kratos data stream --- salt/common/tools/sbin/so-elastic-fleet-integration-policy-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index aa31e9fb6..b3148f7aa 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -62,7 +62,7 @@ echo # Kratos logs echo echo "Setting up Kratos package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "access", "tags": [],"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- rename:\n fields:\n - from: \"audience\"\n to: \"event.dataset\"\n ignore_missing: true\n- add_fields:\n when:\n not: \n has_fields: ['event.dataset']\n target: ''\n fields:\n event.dataset: access", "custom": "pipeline: kratos" }}}}}}' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "kratos", "tags": [],"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- rename:\n fields:\n - from: \"audience\"\n to: \"event.dataset\"\n ignore_missing: true\n- add_fields:\n when:\n not: \n has_fields: ['event.dataset']\n target: ''\n fields:\n event.dataset: access", "custom": "pipeline: kratos" }}}}}}' echo # RITA Logs From c9f458e1e2d7dd1c4fa1e5b24b356ea6c28d4cba Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 25 Jan 2023 08:19:50 -0500 Subject: [PATCH 39/72] Set event.dataset for all Kratos logs to 'access' for now --- salt/elasticsearch/files/ingest/kratos | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/files/ingest/kratos b/salt/elasticsearch/files/ingest/kratos index c55ebd60a..f8dcf53e0 100644 --- a/salt/elasticsearch/files/ingest/kratos +++ b/salt/elasticsearch/files/ingest/kratos @@ -1,6 +1,7 @@ { "description" : "kratos", "processors" : [ + { "set": { "field": "event.dataset", "value": "access" } }, { "pipeline": { "name": "common" } } ] } From 506baa854dd810e2694720a0f354688bef7747fc Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 25 Jan 2023 13:52:54 +0000 Subject: [PATCH 40/72] Configure Elasticsearch output if running Import Mode --- salt/common/tools/sbin/so-elastic-fleet-setup | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index 4e3d7639c..07fb4bc90 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -19,6 +19,12 @@ printf "\n" curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220"]}' printf "\n\n" +{% if grains.role == 'so-import' %} +# Add SO-Manager Elasticsearch Ouput +curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities":[ "/etc/pki/ca.crt" ]}} +printf "\n\n" + +{% else %} # Create Logstash Output payload mkdir -p /opt/so/conf/elastic-fleet/certs cp /etc/ssl/certs/intca.crt /opt/so/conf/elastic-fleet/certs @@ -30,12 +36,13 @@ JSON_STRING=$( jq -n \ --arg LOGSTASHCRT "$LOGSTASHCRT" \ --arg LOGSTASHKEY "$LOGSTASHKEY" \ --arg LOGSTASHCA "$LOGSTASHCA" \ - '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}' + '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}' ) # Add SO-Manager Logstash Ouput curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" +{%- endif %} # Add Elastic Fleet Integrations From 1414b75e0151b6fd2d90352f188540aa5315d577 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 25 Jan 2023 14:07:25 +0000 Subject: [PATCH 41/72] Allow 'elastic-fleet' state to be applied for Import Mode --- salt/top.sls | 1 + setup/so-setup | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/top.sls b/salt/top.sls index e29d3b081..4b8531f4d 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -344,6 +344,7 @@ base: - zeek - schedule - docker_clean + - elastic-fleet '*_receiver and G@saltversion:{{saltversion}}': - match: compound diff --git a/setup/so-setup b/setup/so-setup index 5088395f3..08b16dfb8 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -577,9 +577,9 @@ if ! [[ -f $install_opt_file ]]; then add_web_user info "Restarting SOC to pick up initial user" logCmd "so-soc-restart" - if [[ ! $is_import ]]; then - title "Setting up Elastic Fleet" - logCmd "so-elastic-fleet-setup" + title "Setting up Elastic Fleet" + logCmd "so-elastic-fleet-setup" + if [[ ! $is_import ]]; then title "Setting up Playbook" logCmd "so-playbook-reset" fi From c46b5e734b60898bebe480793a152feb5b23e6fa Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 25 Jan 2023 14:38:23 +0000 Subject: [PATCH 42/72] Add 'elastic-fleet' to the list of allowed states for Import Mode --- salt/allowed_states.map.jinja | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 2f6cc60a0..823b7b647 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -107,7 +107,8 @@ 'zeek', 'schedule', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'elastic-fleet' ], 'so-manager': [ 'salt.master', From 3f99e3402e3f1b6368d9e92117f1c9911be1a272 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 25 Jan 2023 10:53:58 -0500 Subject: [PATCH 43/72] add elasticsearch pillar files to manager and adv_elasticsearch to those that had soc_elasticsearch --- pillar/top.sls | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pillar/top.sls b/pillar/top.sls index d24b6980f..61088e0a3 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -52,6 +52,8 @@ base: - redis.adv_redis - influxdb.soc_influxdb - influxdb.adv_influxdb + - elasticsearch.soc_elasticsearch + - elasticsearch.adv_elasticsearch - backup.soc_backup - backup.adv_backup - minions.{{ grains.id }} @@ -77,6 +79,7 @@ base: - soc_global - kratos.soc_kratos - elasticsearch.soc_elasticsearch + - elasticsearch.adv_elasticsearch - manager.soc_manager - soc.soc_soc - kratos.soc_kratos @@ -112,6 +115,7 @@ base: - influxdb.soc_influxdb - influxdb.adv_influxdb - elasticsearch.soc_elasticsearch + - elasticsearch.adv_elasticsearch - manager.soc_manager - soc.soc_soc - backup.soc_backup @@ -170,6 +174,7 @@ base: {% endif %} - kratos.soc_kratos - elasticsearch.soc_elasticsearch + - elasticsearch.adv_elasticsearch - manager.soc_manager - soc.soc_soc - soc_global From 838beabae52b41bcb3dca7fc4ae78d8ce0c4b680 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 25 Jan 2023 15:58:06 +0000 Subject: [PATCH 44/72] Add missing single quote for Elastic Agent Elasticsearch output --- salt/common/tools/sbin/so-elastic-fleet-setup | 2 +- setup/so-setup | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index 07fb4bc90..d76da08ed 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -21,7 +21,7 @@ printf "\n\n" {% if grains.role == 'so-import' %} # Add SO-Manager Elasticsearch Ouput -curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities":[ "/etc/pki/ca.crt" ]}} +curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities":[ "/etc/pki/ca.crt" ]}}' printf "\n\n" {% else %} diff --git a/setup/so-setup b/setup/so-setup index 08b16dfb8..a9bb2f6b1 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -580,7 +580,7 @@ if ! [[ -f $install_opt_file ]]; then title "Setting up Elastic Fleet" logCmd "so-elastic-fleet-setup" if [[ ! $is_import ]]; then - title "Setting up Playbook" + title "Setting up Playbook" logCmd "so-playbook-reset" fi checkin_at_boot From 31d7e05c457e6ce85c486a42ebd1ad7235d23839 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 25 Jan 2023 10:59:50 -0500 Subject: [PATCH 45/72] refactor so-verify to ensure output streams to console --- setup/so-functions | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index a046b49ca..683a99933 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2383,9 +2383,10 @@ wait_for_salt_minion() { verify_setup() { info "Verifying setup" - output=$(./so-verify "$setup_type" 2>&1) + set -o pipefail + ./so-verify "$setup_type" 2>&1 | tee -a $setup_log result=$? - echo "$output" >> "$setup_log" + set +o pipefail if [[ $result -eq 0 ]]; then whiptail_setup_complete else From 86a925e1c7606a8b3f05757bcb56edad2b583b4e Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 25 Jan 2023 16:09:12 +0000 Subject: [PATCH 46/72] Download Elastic Agent images for Import Mode --- salt/common/tools/sbin/so-image-common | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 3851d8b4a..0f03d891e 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -24,6 +24,8 @@ container_list() { if [ $MANAGERCHECK == 'so-import' ]; then TRUSTED_CONTAINERS=( + "so-elastic-agent" + "so-elastic-agent-builder" "so-elasticsearch" "so-filebeat" "so-idstools" From 9243b01cbb094c495be355ae3578810fb6df6d98 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 25 Jan 2023 11:44:22 -0500 Subject: [PATCH 47/72] Salt for Ubuntu --- salt/common/packages.sls | 28 +++++++++++++++++++++++++++- salt/docker/init.sls | 14 ++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index e52e8174f..34935613e 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -1,6 +1,32 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} -{% if GLOBALS.os != 'CentOS' %} +{% if GLOBALS.os == 'Ubuntu' %} +commonpkgs: + pkg.installed: + - skip_suggestions: True + - pkgs: + - apache2-utils + - wget + - ntpdate + - jq + - python3-docker + - curl + - ca-certificates + - software-properties-common + - apt-transport-https + - openssl + - netcat + - python3-mysqldb + - sqlite3 + - libssl-dev + - python3-dateutil + - python3-m2crypto + - python3-mysqldb + - python3-packaging + - python3-lxml + - git + - vim +{% elif GLOBALS.os == 'Rocky' %} commonpkgs: pkg.installed: - skip_suggestions: True diff --git a/salt/docker/init.sls b/salt/docker/init.sls index f8f89e058..405a75938 100644 --- a/salt/docker/init.sls +++ b/salt/docker/init.sls @@ -4,12 +4,25 @@ # Elastic License 2.0. {% from 'docker/docker.map.jinja' import DOCKER %} +{% from 'vars/globals.map.jinja' import GLOBALS %} + dockergroup: group.present: - name: docker - gid: 920 +{% if GLOBALS.os == 'Ubuntu' %} +dockerheldpackages: + pkg.installed: + - pkgs: + - containerd.io: 1.4.9-1 + - docker-ce: 5:20.10.8~3-0~ubuntu-focal + - docker-ce-cli: 5:20.10.5~3-0~ubuntu-focal + - docker-ce-rootless-extras: 5:20.10.5~3-0~ubuntu-focal + - hold: True + - update_holds: True +{% else %} dockerheldpackages: pkg.installed: - pkgs: @@ -19,6 +32,7 @@ dockerheldpackages: - docker-ce-rootless-extras: 20.10.5-3.el7 - hold: True - update_holds: True +{% endif %} #disable docker from managing iptables iptables_disabled: From 704d99e757751d109c89c8f9294b661f04d85dea Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 25 Jan 2023 11:50:19 -0500 Subject: [PATCH 48/72] Salt for Ubuntu --- salt/common/packages.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index 34935613e..0bb36d07f 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -5,6 +5,7 @@ commonpkgs: pkg.installed: - skip_suggestions: True - pkgs: + - chrony - apache2-utils - wget - ntpdate From 498301b1117eae3f4803a8c1a173a3235b4ea2d3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 25 Jan 2023 12:00:19 -0500 Subject: [PATCH 49/72] Salt for Ubuntu --- setup/so-functions | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index cff943376..26c66736f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2347,7 +2347,12 @@ ubuntu_check() { whiptail_ubuntu_notsupported exit 1 else - whiptail_ubuntu_warning + if [[ $UBUNTUINSTALL == "needtoupgrade" ]]; then + whiptail_ubuntu_warning + else + whiptail_ubuntu_notsupported + exit 1 + fi fi fi } From c3717dae672927b327016da0a2cde05ac7c62fae Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 25 Jan 2023 18:27:00 +0000 Subject: [PATCH 50/72] Add Elastic Fleet firewall configuration for Import Mode --- salt/firewall/assigned_hostgroups.map.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 6e249ec93..3b6f4a59a 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -436,6 +436,7 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} + - {{ portgroups.elastic_agent_control }} sensors: portgroups: - {{ portgroups.beats_5044 }} @@ -453,6 +454,10 @@ role: elasticsearch_rest: portgroups: - {{ portgroups.elasticsearch_rest }} + elastic_agent_endpoint: + portgroups: + - {{ portgroups.elasticsearch_rest }} + - {{ portgroups.elastic_agent_control }} analyst: portgroups: - {{ portgroups.nginx }} From b7a5937dc1675e61461e6bcd08e40ae39e46f7c2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 25 Jan 2023 14:04:36 -0500 Subject: [PATCH 51/72] add soc_logstash and adv_logstash to nodes in pillar/top --- pillar/top.sls | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pillar/top.sls b/pillar/top.sls index 61088e0a3..87cab8c29 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -24,11 +24,15 @@ base: - logstash - logstash.manager - logstash.search + - logstash.soc_logstash + - logstash.adv_logstash - elasticsearch.index_templates '*_manager': - logstash - logstash.manager + - logstash.soc_logstash + - logstash.adv_logstash - elasticsearch.index_templates '*_manager or *_managersearch': @@ -98,6 +102,7 @@ base: - logstash.manager - logstash.search - logstash.soc_logstash + - logstash.adv_logstash - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth @@ -139,6 +144,8 @@ base: '*_searchnode': - logstash - logstash.search + - logstash.soc_logstash + - logstash.adv_logstash - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth @@ -153,6 +160,8 @@ base: '*_receiver': - logstash - logstash.receiver + - logstash.soc_logstash + - logstash.adv_logstash {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} From 5c58cda872da93c8e4179e912283ad241b18136f Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 25 Jan 2023 19:29:50 +0000 Subject: [PATCH 52/72] Move certificate configuration outside of conditional logic --- salt/common/tools/sbin/so-elastic-fleet-setup | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index d76da08ed..01968fcb4 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -19,6 +19,11 @@ printf "\n" curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220"]}' printf "\n\n" +# Configure certificates +mkdir -p /opt/so/conf/elastic-fleet/certs +cp /etc/ssl/certs/intca.crt /opt/so/conf/elastic-fleet/certs +cp /etc/pki/elasticfleet* /opt/so/conf/elastic-fleet/certs + {% if grains.role == 'so-import' %} # Add SO-Manager Elasticsearch Ouput curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities":[ "/etc/pki/ca.crt" ]}}' @@ -26,9 +31,6 @@ printf "\n\n" {% else %} # Create Logstash Output payload -mkdir -p /opt/so/conf/elastic-fleet/certs -cp /etc/ssl/certs/intca.crt /opt/so/conf/elastic-fleet/certs -cp /etc/pki/elasticfleet* /opt/so/conf/elastic-fleet/certs LOGSTASHCRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/elasticfleet.crt) LOGSTASHKEY=$(openssl rsa -in /opt/so/conf/elastic-fleet/certs/elasticfleet.key) LOGSTASHCA=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/intca.crt) From 790aa6b684e292d9fb17bc64f33c92caf963aa93 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 25 Jan 2023 15:18:56 -0500 Subject: [PATCH 53/72] add logstash pillar items for minions --- salt/common/tools/sbin/so-minion | 19 +++++++++++++++++++ setup/so-functions | 19 +++++-------------- setup/so-setup | 3 +++ 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/salt/common/tools/sbin/so-minion b/salt/common/tools/sbin/so-minion index 1f71eb12e..a51a31ed4 100755 --- a/salt/common/tools/sbin/so-minion +++ b/salt/common/tools/sbin/so-minion @@ -119,6 +119,18 @@ function add_elastic_to_minion() { " " >> $PILLARFILE } +function add_logstash_to_minion() { + # Create the logstash advanced pillar + printf '%s\n'\ + "logstash_settings:"\ + " ls_host: '$LSHOSTNAME'"\ + " ls_pipeline_batch_size: 125"\ + " ls_input_threads: 1"\ + " lsheap: $LSHEAP"\ + " ls_pipeline_workers: $CPUCORES"\ + " " >> $PILLARFILE +} + # Analyst Workstation function add_analyst_to_minion() { printf '%s\n'\ @@ -167,6 +179,7 @@ function add_sensor_to_minion() { function createEVAL() { add_elastic_to_minion + add_logstash_to_minion add_sensor_to_minion } @@ -176,20 +189,24 @@ function createIDHNODE() { function createIMPORT() { add_elastic_to_minion + add_logstash_to_minion add_sensor_to_minion } function createHEAVYNODE() { add_elastic_to_minion + add_logstash_to_minion add_sensor_to_minion } function createMANAGER() { add_elastic_to_minion + add_logstash_to_minion } function createMANAGERSEARCH() { add_elastic_to_minion + add_logstash_to_minion } function createSENSOR() { @@ -198,10 +215,12 @@ function createSENSOR() { function createSEARCHNODE() { add_elastic_to_minion + add_logstash_to_minion } function createSTANDALONE() { add_elastic_to_minion + add_logstash_to_minion add_sensor_to_minion } diff --git a/setup/so-functions b/setup/so-functions index a046b49ca..045a728aa 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1307,15 +1307,7 @@ idh_pillar() { logstash_pillar() { # Create the logstash advanced pillar touch $adv_logstash_pillar_file - title "Create the logstash pillar" - printf '%s\n'\ - "logstash_settings:"\ - " ls_host: '$HOSTNAME'"\ - " ls_pipeline_batch_size: 125"\ - " ls_input_threads: 1"\ - " lsheap: $NODE_LS_HEAP_SIZE"\ - " ls_pipeline_workers: $num_cpu_cores"\ - "" > "$logstash_pillar_file" + touch $logstash_pillar_file } # Set Logstash heap size based on total memory @@ -1339,10 +1331,6 @@ ls_heapsize() { esac export LS_HEAP_SIZE - if [[ "$install_type" =~ ^(EVAL|MANAGERSEARCH|STANDALONE)$ ]]; then - NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE - export NODE_LS_HEAP_SIZE - fi } idstools_pillar() { @@ -1887,6 +1875,9 @@ drop_install_options() { NODETYPE=${install_type^^} echo "NODETYPE=$NODETYPE" >> /opt/so/install.txt echo "CORECOUNT=$lb_procs" >> /opt/so/install.txt + echo "LSHOSTNAME=$HOSTNAME" >> /opt/so/install.txt + echo "LSHEAP=$LS_HEAP_SIZE" >> /opt/so/install.txt + echo "CPUCORES=$num_cpu_cores" >> /opt/so/install.txt } remove_package() { @@ -2391,4 +2382,4 @@ verify_setup() { else whiptail_setup_failed fi -} \ No newline at end of file +} diff --git a/setup/so-setup b/setup/so-setup index 2644ca220..ccae9510b 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -528,6 +528,9 @@ if ! [[ -f $install_opt_file ]]; then export PATCHSCHEDULENAME=$PATCHSCHEDULENAME export INTERFACE="bond0" export CORECOUNT=$lb_procs + export LSHOSTNAME=$HOSTNAME + export LSHEAP=$LS_HEAP_SIZE + export CPUCORES=$num_cpu_cores logCmd "so-minion -o=setup" title "Creating Global SLS" From 6ee66a34bce56d5db19dab5f6be42be10037a8af Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 25 Jan 2023 17:12:03 -0500 Subject: [PATCH 54/72] Revert "Elastic Agent and Fleet - Import Mode" --- salt/allowed_states.map.jinja | 3 +-- salt/common/tools/sbin/so-elastic-fleet-setup | 13 ++----------- salt/common/tools/sbin/so-image-common | 2 -- salt/firewall/assigned_hostgroups.map.yaml | 5 ----- salt/top.sls | 1 - setup/so-setup | 8 ++++---- 6 files changed, 7 insertions(+), 25 deletions(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 823b7b647..2f6cc60a0 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -107,8 +107,7 @@ 'zeek', 'schedule', 'tcpreplay', - 'docker_clean', - 'elastic-fleet' + 'docker_clean' ], 'so-manager': [ 'salt.master', diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index 01968fcb4..4e3d7639c 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -19,18 +19,10 @@ printf "\n" curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220"]}' printf "\n\n" -# Configure certificates +# Create Logstash Output payload mkdir -p /opt/so/conf/elastic-fleet/certs cp /etc/ssl/certs/intca.crt /opt/so/conf/elastic-fleet/certs cp /etc/pki/elasticfleet* /opt/so/conf/elastic-fleet/certs - -{% if grains.role == 'so-import' %} -# Add SO-Manager Elasticsearch Ouput -curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities":[ "/etc/pki/ca.crt" ]}}' -printf "\n\n" - -{% else %} -# Create Logstash Output payload LOGSTASHCRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/elasticfleet.crt) LOGSTASHKEY=$(openssl rsa -in /opt/so/conf/elastic-fleet/certs/elasticfleet.key) LOGSTASHCA=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/intca.crt) @@ -38,13 +30,12 @@ JSON_STRING=$( jq -n \ --arg LOGSTASHCRT "$LOGSTASHCRT" \ --arg LOGSTASHKEY "$LOGSTASHKEY" \ --arg LOGSTASHCA "$LOGSTASHCA" \ - '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}' + '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}' ) # Add SO-Manager Logstash Ouput curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" -{%- endif %} # Add Elastic Fleet Integrations diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 0f03d891e..3851d8b4a 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -24,8 +24,6 @@ container_list() { if [ $MANAGERCHECK == 'so-import' ]; then TRUSTED_CONTAINERS=( - "so-elastic-agent" - "so-elastic-agent-builder" "so-elasticsearch" "so-filebeat" "so-idstools" diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 3b6f4a59a..6e249ec93 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -436,7 +436,6 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} - - {{ portgroups.elastic_agent_control }} sensors: portgroups: - {{ portgroups.beats_5044 }} @@ -454,10 +453,6 @@ role: elasticsearch_rest: portgroups: - {{ portgroups.elasticsearch_rest }} - elastic_agent_endpoint: - portgroups: - - {{ portgroups.elasticsearch_rest }} - - {{ portgroups.elastic_agent_control }} analyst: portgroups: - {{ portgroups.nginx }} diff --git a/salt/top.sls b/salt/top.sls index 4b8531f4d..e29d3b081 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -344,7 +344,6 @@ base: - zeek - schedule - docker_clean - - elastic-fleet '*_receiver and G@saltversion:{{saltversion}}': - match: compound diff --git a/setup/so-setup b/setup/so-setup index 1d5e1c122..86e9f23d0 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -587,10 +587,10 @@ if ! [[ -f $install_opt_file ]]; then add_web_user info "Restarting SOC to pick up initial user" logCmd "so-soc-restart" - title "Setting up Elastic Fleet" - logCmd "so-elastic-fleet-setup" - if [[ ! $is_import ]]; then - title "Setting up Playbook" + if [[ ! $is_import ]]; then + title "Setting up Elastic Fleet" + logCmd "so-elastic-fleet-setup" + title "Setting up Playbook" logCmd "so-playbook-reset" fi checkin_at_boot From c57d390bacd2e8756783993fbb956b04fa8e56bd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 25 Jan 2023 17:40:40 -0500 Subject: [PATCH 55/72] Proxy Stuff --- setup/so-functions | 8 +------- setup/so-setup | 4 ++++ setup/so-whiptail | 31 ++++++------------------------- 3 files changed, 11 insertions(+), 32 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 26c66736f..c28a1c6c7 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -383,14 +383,8 @@ collect_mngr_hostname() { fi } -collect_net_method() { +collect_net_method() { whiptail_net_method - - if [[ "$network_traffic" == *"_MANAGER" ]]; then - whiptail_manager_updates_warning - MANAGERUPDATES=1 - fi - if [[ "$network_traffic" == "PROXY"* ]]; then collect_proxy no_ask fi diff --git a/setup/so-setup b/setup/so-setup index bbc3ae971..320f774b2 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -319,6 +319,7 @@ if ! [[ -f $install_opt_file ]]; then check_elastic_license check_requirements "manager" networking_needful + collect_net_method collect_dockernet if [[ $is_iso ]]; then whiptail_airgap @@ -342,6 +343,7 @@ if ! [[ -f $install_opt_file ]]; then check_elastic_license check_requirements "manager" networking_needful + collect_net_method collect_dockernet if [[ $is_iso ]]; then whiptail_airgap @@ -364,6 +366,7 @@ if ! [[ -f $install_opt_file ]]; then ubuntu_check check_requirements "manager" networking_needful + collect_net_method collect_dockernet if [[ $is_iso ]]; then whiptail_airgap @@ -384,6 +387,7 @@ if ! [[ -f $install_opt_file ]]; then ubuntu_check check_requirements "manager" networking_needful + collect_net_method collect_dockernet if [[ $is_iso ]]; then whiptail_airgap diff --git a/setup/so-whiptail b/setup/so-whiptail index 5615edf41..e5de2cf5b 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -816,34 +816,15 @@ whiptail_net_method() { ) local proxy_desc="proxy the traffic for git, docker client, wget, curl, ${pkg_mngr}, and various other SO components through a separate server in your environment." - if [[ $is_minion ]]; then - read -r -d '' options_msg <<- EOM + read -r -d '' options_msg <<- EOM ${options_msg} - - "Direct + Manager" - all traffic passes to the Internet normally, but ${pkg_mngr} updates will instead be pulled from ${mngr_article} manager. - "Proxy" - ${proxy_desc} + EOM + options+=( + " Proxy " "" + ) - "Proxy + Manager" - proxy all traffic from the "Proxy" option except ${pkg_mngr} updates, which will instead pull from the manager. - EOM - - options+=( - " Direct + Manager " "" - " Proxy " "" - " Proxy + Manager " "" - ) - local height=25 - else - read -r -d '' options_msg <<- EOM - ${options_msg} - - "Proxy" - ${proxy_desc} - EOM - options+=( - " Proxy " "" - ) - local height=17 - fi + local height=17 local msg read -r -d '' msg <<- EOM From 9e146184d66ad4ef271744d7febff3b32a15cf5c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 25 Jan 2023 17:43:02 -0500 Subject: [PATCH 56/72] Proxy Stuff --- setup/so-setup | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-setup b/setup/so-setup index 320f774b2..8ef8a102b 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -142,6 +142,7 @@ if [[ -f /root/accept_changes ]]; then mv "$setup_log" "$setup_log.bak" [ -f "$error_log" ] && mv "$error_log" "$error_log.bak" reinstall_init + reset_proxy fi title "Parsing Username for Install" From 08750154b42a64272939bfefb9ab0d30ec44fe92 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 26 Jan 2023 09:11:28 -0500 Subject: [PATCH 57/72] add missing quotes in check_web_pass --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 045a728aa..666721557 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -200,7 +200,7 @@ check_service_status() { } check_web_pass() { - info Making sure web credential passwords match + info "Making sure web credential passwords match" check_pass_match "$WEBPASSWD1" "$WEBPASSWD2" "WPMATCH" } From 2456aac31107d952d1bf69b38fe83e904ee6cd24 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 26 Jan 2023 09:57:44 -0500 Subject: [PATCH 58/72] Proxy Stuff --- setup/so-functions | 1 + setup/so-setup | 3 +++ 2 files changed, 4 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index c28a1c6c7..d92cc8efa 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -387,6 +387,7 @@ collect_net_method() { whiptail_net_method if [[ "$network_traffic" == "PROXY"* ]]; then collect_proxy no_ask + needs_proxy=true fi } diff --git a/setup/so-setup b/setup/so-setup index 8ef8a102b..32feb8f9a 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -490,6 +490,9 @@ if ! [[ -f $install_opt_file ]]; then percentage=0 es_heapsize ls_heapsize + if [[ $needs_proxy ]]; then + set_proxy + fi set_redirect # Generate Interface Vars generate_interface_vars From aafbdf6afcca03df8706590ff3282520cea43b52 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 26 Jan 2023 10:12:37 -0500 Subject: [PATCH 59/72] adjust retry and timeout for wait_for_influxdb --- salt/influxdb/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 5a4936843..1758f17ae 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -94,10 +94,10 @@ wait_for_influxdb: - ssl: True - verify_ssl: False - status: 200 - - timeout: 30 + - timeout: 10 - retry: - attempts: 5 - interval: 60 + attempts: 20 + interval: 5 - require: - docker_container: so-influxdb From 1fb6cf7bfeaa3d6777b1683be3117dc4d959dbd9 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 15:59:49 +0000 Subject: [PATCH 60/72] Add Elastic Fleet to allowed states for Import Mode --- salt/allowed_states.map.jinja | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 2f6cc60a0..823b7b647 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -107,7 +107,8 @@ 'zeek', 'schedule', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'elastic-fleet' ], 'so-manager': [ 'salt.master', From 6a803dfe356cf4524f47dcffe8c5e80bacdfaf20 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 16:01:03 +0000 Subject: [PATCH 61/72] Add Elastic Fleet to top file configuration for Import Mode --- salt/top.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/top.sls b/salt/top.sls index e29d3b081..4b8531f4d 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -344,6 +344,7 @@ base: - zeek - schedule - docker_clean + - elastic-fleet '*_receiver and G@saltversion:{{saltversion}}': - match: compound From a9a119f1ab820a6b97901f32f94fcf355cc9a31c Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 16:02:27 +0000 Subject: [PATCH 62/72] Add Elasticsearch output to 'so-elastic-fleet-setup' for Import Mode --- salt/common/tools/sbin/so-elastic-fleet-setup | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index 4e3d7639c..c945aeec3 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -19,10 +19,22 @@ printf "\n" curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220"]}' printf "\n\n" -# Create Logstash Output payload +# Configure certificates mkdir -p /opt/so/conf/elastic-fleet/certs cp /etc/ssl/certs/intca.crt /opt/so/conf/elastic-fleet/certs cp /etc/pki/elasticfleet* /opt/so/conf/elastic-fleet/certs + +{% if grains.role == 'so-import' %} +# Add SO-Manager Elasticsearch Ouput +ESCACRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/intca.crt) +JSON_STRING=$( jq -n \ + --arg ESCACRT "$ESCACRT" \ + '{"name":"so-manager_elasticsearch2","id":"so-manager_elasticsearch2","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' ) +curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +printf "\n\n" + +{% else %} +# Create Logstash Output payload LOGSTASHCRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/elasticfleet.crt) LOGSTASHKEY=$(openssl rsa -in /opt/so/conf/elastic-fleet/certs/elasticfleet.key) LOGSTASHCA=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/intca.crt) @@ -30,12 +42,13 @@ JSON_STRING=$( jq -n \ --arg LOGSTASHCRT "$LOGSTASHCRT" \ --arg LOGSTASHKEY "$LOGSTASHKEY" \ --arg LOGSTASHCA "$LOGSTASHCA" \ - '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}' + '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}' ) # Add SO-Manager Logstash Ouput curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" +{%- endif %} # Add Elastic Fleet Integrations From 8051fc70eba77529b9fba1ae60a16dab0fa488e4 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 16:03:59 +0000 Subject: [PATCH 63/72] Temporarily disable the loading of the RITA package policy --- .../tools/sbin/so-elastic-fleet-integration-policy-load | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index b3148f7aa..f409d9a17 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -66,10 +66,10 @@ curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POS echo # RITA Logs -echo -echo "Setting up RITA package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita\n- if:\n log.file.path: beacons.csv\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.beacon\n- if:\n regexp:\n log.file.path: \"*connections.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.connection\n- if:\n log.file.path: \"exploded-dns.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.dns" }}}}}}' -echo +#echo +#echo "Setting up RITA package policy..." +#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita\n- if:\n log.file.path: beacons.csv\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.beacon\n- if:\n regexp:\n log.file.path: \"*connections.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.connection\n- if:\n log.file.path: \"exploded-dns.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.dns" }}}}}}' +#echo # Elasticsearch logs echo From 43ffcb1d6382a4ac2d05b281e57d5a99bad0a8c2 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 16:05:16 +0000 Subject: [PATCH 64/72] Allow setup to set up Elastic Fleet for Import Mode --- setup/so-setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 86e9f23d0..8aa34d04f 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -587,9 +587,9 @@ if ! [[ -f $install_opt_file ]]; then add_web_user info "Restarting SOC to pick up initial user" logCmd "so-soc-restart" + title "Setting up Elastic Fleet" + logCmd "so-elastic-fleet-setup" if [[ ! $is_import ]]; then - title "Setting up Elastic Fleet" - logCmd "so-elastic-fleet-setup" title "Setting up Playbook" logCmd "so-playbook-reset" fi From 7d68ef0e8b676236e9ccb9913ce18480d7c6015b Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 16:07:31 +0000 Subject: [PATCH 65/72] Add Elastic Agent and Fleet to firewall configuration for Import Mode --- salt/firewall/assigned_hostgroups.map.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 6e249ec93..0f16228b2 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -436,6 +436,7 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} + - {{ portgroups.elastic_agent_control }} sensors: portgroups: - {{ portgroups.beats_5044 }} @@ -453,6 +454,10 @@ role: elasticsearch_rest: portgroups: - {{ portgroups.elasticsearch_rest }} + elastic_agent_endpoint: + portgroups: + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} analyst: portgroups: - {{ portgroups.nginx }} From f1db1bc2733b57e4b84c5abaa3312e037fd85bb6 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 16:12:06 +0000 Subject: [PATCH 66/72] Ensure Kratos events are sent to a data stream instead of an index --- salt/common/tools/sbin/so-image-common | 4 +++- salt/elasticsearch/defaults.yaml | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 3851d8b4a..aa2292b22 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -36,7 +36,9 @@ container_list() { "so-steno" "so-suricata" "so-telegraf" - "so-zeek" + "so-zeek" + "so-elastic-agent" + "so-elastic-agent-builder" ) elif [ $MANAGERCHECK != 'so-helix' ]; then TRUSTED_CONTAINERS=( diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 11b445c62..3d1182255 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -2677,6 +2677,9 @@ elasticsearch: delete: 365 index_sorting: False index_template: + data_stream: + hidden: false + allow_custom_routing: false index_patterns: - logs-kratos-so* template: From b381c5424e38439448fb3574e568a3d7eac39adf Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 16:13:23 +0000 Subject: [PATCH 67/72] Remove extra whitespace after 'so-elastic-agent-builder' line in 'so-image-common' --- salt/common/tools/sbin/so-image-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index aa2292b22..945d3f4ed 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -38,7 +38,7 @@ container_list() { "so-telegraf" "so-zeek" "so-elastic-agent" - "so-elastic-agent-builder" + "so-elastic-agent-builder" ) elif [ $MANAGERCHECK != 'so-helix' ]; then TRUSTED_CONTAINERS=( From 25a6eba1662790e3dc868ac6497d3877dcea2495 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 26 Jan 2023 12:30:35 -0500 Subject: [PATCH 68/72] Update so-verify --- setup/so-verify | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-verify b/setup/so-verify index 77cdf7c8e..44c0465ff 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -36,6 +36,7 @@ log_has_errors() { grep -vE "The Salt Master has cached the public key for this node" | \ grep -vE "Minion failed to authenticate with the master" | \ grep -vE "Failed to connect to ::1" | \ + grep -vE "Failed to set locale" | \ grep -vE "perl-Error-" | \ grep -vE "Failed:\s*?[0-9]+" | \ grep -vE "Status .* was not found" | \ From b3123f7895951675411150ce3d172120a0005363 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 17:57:07 +0000 Subject: [PATCH 69/72] Remove unnecessary Logstash pipelines from the pillar --- pillar/logstash/manager.sls | 4 +--- pillar/logstash/receiver.sls | 5 ++--- pillar/logstash/search.sls | 11 ----------- 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls index cfeb0a6ae..41a2197fd 100644 --- a/pillar/logstash/manager.sls +++ b/pillar/logstash/manager.sls @@ -2,9 +2,7 @@ logstash: pipelines: manager: config: - - so/0009_input_beats.conf - - so/0010_input_hhbeats.conf - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf - so/9999_output_redis.conf.jinja - \ No newline at end of file + diff --git a/pillar/logstash/receiver.sls b/pillar/logstash/receiver.sls index 09c2549e6..4d0637dde 100644 --- a/pillar/logstash/receiver.sls +++ b/pillar/logstash/receiver.sls @@ -2,8 +2,7 @@ logstash: pipelines: receiver: config: - - so/0009_input_beats.conf - - so/0010_input_hhbeats.conf - so/0011_input_endgame.conf + - so/0012_input_elastic_agent.conf - so/9999_output_redis.conf.jinja - \ No newline at end of file + diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index fb10d18e7..0b660b7ef 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -3,16 +3,5 @@ logstash: search: config: - so/0900_input_redis.conf.jinja - - so/9000_output_zeek.conf.jinja - - so/9002_output_import.conf.jinja - - so/9034_output_syslog.conf.jinja - - so/9050_output_filebeatmodules.conf.jinja - - so/9100_output_osquery.conf.jinja - - so/9400_output_suricata.conf.jinja - - so/9500_output_beats.conf.jinja - - so/9600_output_ossec.conf.jinja - - so/9700_output_strelka.conf.jinja - - so/9800_output_logscan.conf.jinja - - so/9801_output_rita.conf.jinja - so/9805_output_elastic_agent.conf.jinja - so/9900_output_endgame.conf.jinja From e4271043c6b7aae64aa92f20a7d0c7c552026aae Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 26 Jan 2023 18:05:14 +0000 Subject: [PATCH 70/72] Remove unnecessary Logstash pipelines --- .../pipelines/config/so/0009_input_beats.conf | 11 - .../config/so/0010_input_hhbeats.conf | 40 ---- .../pipelines/config/so/0800_input_eval.conf | 204 ------------------ .../config/so/0899_input_minio.conf.jinja | 23 -- .../config/so/9000_output_zeek.conf.jinja | 13 -- .../config/so/9002_output_import.conf.jinja | 13 -- .../config/so/9034_output_syslog.conf.jinja | 13 -- .../so/9050_output_filebeatmodules.conf.jinja | 14 -- .../config/so/9100_output_osquery.conf.jinja | 13 -- .../config/so/9200_output_firewall.conf.jinja | 12 -- .../config/so/9400_output_suricata.conf.jinja | 13 -- .../config/so/9500_output_beats.conf.jinja | 26 --- .../config/so/9600_output_ossec.conf.jinja | 13 -- .../config/so/9700_output_strelka.conf.jinja | 14 -- .../config/so/9800_output_logscan.conf.jinja | 14 -- .../config/so/9801_output_rita.conf.jinja | 13 -- 16 files changed, 449 deletions(-) delete mode 100644 salt/logstash/pipelines/config/so/0009_input_beats.conf delete mode 100644 salt/logstash/pipelines/config/so/0010_input_hhbeats.conf delete mode 100644 salt/logstash/pipelines/config/so/0800_input_eval.conf delete mode 100644 salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9002_output_import.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja diff --git a/salt/logstash/pipelines/config/so/0009_input_beats.conf b/salt/logstash/pipelines/config/so/0009_input_beats.conf deleted file mode 100644 index 8643a64b4..000000000 --- a/salt/logstash/pipelines/config/so/0009_input_beats.conf +++ /dev/null @@ -1,11 +0,0 @@ -input { - beats { - port => "5044" - tags => [ "beat-ext" ] - } -} -filter { - mutate { - rename => {"@metadata" => "metadata"} - } -} diff --git a/salt/logstash/pipelines/config/so/0010_input_hhbeats.conf b/salt/logstash/pipelines/config/so/0010_input_hhbeats.conf deleted file mode 100644 index 050d01d73..000000000 --- a/salt/logstash/pipelines/config/so/0010_input_hhbeats.conf +++ /dev/null @@ -1,40 +0,0 @@ -input { - beats { - port => "5644" - ssl => true - ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"] - ssl_certificate => "/usr/share/logstash/filebeat.crt" - ssl_key => "/usr/share/logstash/filebeat.key" - #tags => [ "beat" ] - } -} -filter { - if [type] == "ids" or [type] =~ "bro" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_field => { "sensor_name" => "%{[beat][name]}" } - add_field => { "syslog-host_from" => "%{[beat][name]}" } - remove_field => [ "beat", "prospector", "input", "offset" ] - } - } - if [type] =~ "ossec" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_field => { "syslog-host_from" => "%{[beat][name]}" } - remove_field => [ "beat", "prospector", "input", "offset" ] - } - } - if [type] == "osquery" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_tag => ["osquery"] - } - json { - source => "message" - target => "osquery" - } - } -} diff --git a/salt/logstash/pipelines/config/so/0800_input_eval.conf b/salt/logstash/pipelines/config/so/0800_input_eval.conf deleted file mode 100644 index 35a977d04..000000000 --- a/salt/logstash/pipelines/config/so/0800_input_eval.conf +++ /dev/null @@ -1,204 +0,0 @@ -# Updated by: Mike Reeves -# Last Update: 11/1/2018 - -input { - file { - path => "/suricata/eve.json" - type => "ids" - add_field => { "engine" => "suricata" } - } - file { - path => "/nsm/zeek/logs/current/conn*.log" - type => "zeek.conn" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dce_rpc*.log" - type => "zeek.dce_rpc" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dhcp*.log" - type => "zeek.dhcp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dnp3*.log" - type => "zeek.dnp3" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dns*.log" - type => "zeek.dns" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dpd*.log" - type => "zeek.dpd" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/files*.log" - type => "zeek.files" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ftp*.log" - type => "zeek.ftp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/http*.log" - type => "zeek.http" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/intel*.log" - type => "zeek.intel" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/irc*.log" - type => "zeek.irc" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/kerberos*.log" - type => "zeek.kerberos" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/modbus*.log" - type => "zeek.modbus" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/mysql*.log" - type => "zeek.mysql" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/notice*.log" - type => "zeek.notice" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ntlm*.log" - type => "zeek.ntlm" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/pe*.log" - type => "zeek.pe" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/radius*.log" - type => "zeek.radius" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/rdp*.log" - type => "zeek.rdp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/rfb*.log" - type => "zeek.rfb" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/signatures*.log" - type => "zeek.signatures" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/sip*.log" - type => "zeek.sip" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/smb_files*.log" - type => "zeek.smb_files" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/smb_mapping*.log" - type => "zeek.smb_mapping" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/smtp*.log" - type => "zeek.smtp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/snmp*.log" - type => "zeek.snmp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/socks*.log" - type => "zeek.socks" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/software*.log" - type => "zeek.software" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ssh*.log" - type => "zeek.ssh" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ssl*.log" - type => "zeek.ssl" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/syslog*.log" - type => "zeek.syslog" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/tunnel*.log" - type => "zeek.tunnels" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/weird*.log" - type => "zeek.weird" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/x509*.log" - type => "zeek.x509" - tags => ["zeek"] - } - file { - path => "/wazuh/alerts/alerts.json" - type => "ossec" - } -# file { -# path => "/wazuh/archives/archives.json" -# type => "ossec_archive" -# } - file { - path => "/osquery/logs/result.log" - type => "osquery" - tags => ["osquery"] - } - file { - path => "/strelka/strelka.log" - type => "strelka" - } -} -filter { - if "import" in [tags] { - mutate { - #add_tag => [ "conf_file_0007"] - } - } -} diff --git a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja deleted file mode 100644 index 7a0848b39..000000000 --- a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja +++ /dev/null @@ -1,23 +0,0 @@ -{%- if grains.role == 'so-heavynode' %} -{%- set HOST = GLOBALS.hostname %} -{%- else %} -{%- set HOST = GLOBALS.manager %} -{% endif -%} -{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} -{%- set access_key = salt['pillar.get']('minio:access_key', '') %} -{%- set access_secret = salt['pillar.get']('minio:access_secret', '') %} -{%- set INTERVAL = salt['pillar.get']('s3_settings:interval', 5) %} -input { - s3 { - access_key_id => "{{ access_key }}" - secret_access_key => "{{ access_secret }}" - endpoint => "https://{{ HOST }}:9595" - bucket => "logstash" - delete => true - interval => {{ INTERVAL }} - codec => json - additional_settings => { - "force_path_style" => true - } - } -} diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja deleted file mode 100644 index 7b8c03f45..000000000 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "zeek" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-zeek" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja deleted file mode 100644 index a57830229..000000000 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if "import" in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-import" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja deleted file mode 100644 index 4c49c61ea..000000000 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "syslog" { - elasticsearch { - pipeline => "%{module}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-syslog" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja deleted file mode 100644 index 672a83876..000000000 --- a/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja +++ /dev/null @@ -1,14 +0,0 @@ -output { - if "filebeat" in [metadata][pipeline] { - elasticsearch { - id => "filebeat_modules_metadata_pipeline" - pipeline => "%{[metadata][pipeline]}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-%{[event][module]}-%{+YYYY.MM.dd}" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja deleted file mode 100644 index 8dbea872e..000000000 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "osquery" and "live_query" not in [dataset] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-osquery" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja deleted file mode 100644 index 7942aa50c..000000000 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ /dev/null @@ -1,12 +0,0 @@ -output { - if [dataset] =~ "firewall" { - elasticsearch { - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-firewall" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja deleted file mode 100644 index 13df33e16..000000000 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "suricata" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-ids" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja deleted file mode 100644 index b4aafecad..000000000 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ /dev/null @@ -1,26 +0,0 @@ -output { - if "beat-ext" in [tags] and "import" not in [tags] and "filebeat" not in [metadata][pipeline] { - if [metadata][_id] { - elasticsearch { - pipeline => "beats.common" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-beats" - ssl => true - ssl_certificate_verification => false - document_id => "%{[metadata][_id]}" - } - } else { - elasticsearch { - pipeline => "beats.common" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-beats" - ssl => true - ssl_certificate_verification => false - } - } - } -} diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja deleted file mode 100644 index ca3eeb6c1..000000000 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "ossec" { - elasticsearch { - pipeline => "%{module}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-ossec" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja deleted file mode 100644 index 281cdda5b..000000000 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ /dev/null @@ -1,14 +0,0 @@ -output { - if [module] =~ "strelka" { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-strelka" - ssl => true - ssl_certificate_verification => false - } - } -} - diff --git a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja deleted file mode 100644 index 8127de23a..000000000 --- a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja +++ /dev/null @@ -1,14 +0,0 @@ -output { - if [module] =~ "logscan" { - elasticsearch { - id => "logscan_pipeline" - pipeline => "logscan.alert" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-logscan" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja b/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja deleted file mode 100644 index 7f9d795e6..000000000 --- a/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "rita" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-rita" - ssl => true - ssl_certificate_verification => false - } - } -} From a27fc5c768b9eedf6e76b807ec7c7b0759f8c4fb Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 26 Jan 2023 16:17:36 -0500 Subject: [PATCH 71/72] Update to Elastic 8.6.1 --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index 409bc9209..3ac67ed27 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.5.3","id": "8.5.3","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.6.1","id": "8.6.1","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From 86ca51ff99c5f4285b9ea87f3522ef40ce7d63ea Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 26 Jan 2023 16:18:06 -0500 Subject: [PATCH 72/72] Update to Elastic 8.6.1 --- salt/kibana/bin/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index dadb48ffe..2a4e85b9d 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.5.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.6.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done