diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 241245044..3ce481ee0 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -1,6 +1,12 @@ name: python-test -on: [push, pull_request] +on: + push: + paths: + - "salt/sensoroni/files/analyzers/**" + pull_request: + paths: + - "salt/sensoroni/files/analyzers/**" jobs: build: diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls index cfeb0a6ae..41a2197fd 100644 --- a/pillar/logstash/manager.sls +++ b/pillar/logstash/manager.sls @@ -2,9 +2,7 @@ logstash: pipelines: manager: config: - - so/0009_input_beats.conf - - so/0010_input_hhbeats.conf - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf - so/9999_output_redis.conf.jinja - \ No newline at end of file + diff --git a/pillar/logstash/receiver.sls b/pillar/logstash/receiver.sls index 09c2549e6..4d0637dde 100644 --- a/pillar/logstash/receiver.sls +++ b/pillar/logstash/receiver.sls @@ -2,8 +2,7 @@ logstash: pipelines: receiver: config: - - so/0009_input_beats.conf - - so/0010_input_hhbeats.conf - so/0011_input_endgame.conf + - so/0012_input_elastic_agent.conf - so/9999_output_redis.conf.jinja - \ No newline at end of file + diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index fb10d18e7..0b660b7ef 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -3,16 +3,5 @@ logstash: search: config: - so/0900_input_redis.conf.jinja - - so/9000_output_zeek.conf.jinja - - so/9002_output_import.conf.jinja - - so/9034_output_syslog.conf.jinja - - so/9050_output_filebeatmodules.conf.jinja - - so/9100_output_osquery.conf.jinja - - so/9400_output_suricata.conf.jinja - - so/9500_output_beats.conf.jinja - - so/9600_output_ossec.conf.jinja - - so/9700_output_strelka.conf.jinja - - so/9800_output_logscan.conf.jinja - - so/9801_output_rita.conf.jinja - so/9805_output_elastic_agent.conf.jinja - so/9900_output_endgame.conf.jinja diff --git a/pillar/node_data/ips.sls b/pillar/node_data/ips.sls index 233038e66..59c598879 100644 --- a/pillar/node_data/ips.sls +++ b/pillar/node_data/ips.sls @@ -1,7 +1,5 @@ {% set node_types = {} %} {% set manage_alived = salt.saltutil.runner('manage.alived', show_ip=True) %} -{% set manager = grains.master %} -{% set manager_type = manager.split('_')|last %} {% for minionid, ip in salt.saltutil.runner('mine.get', tgt='*', fun='network.ip_addrs', tgt_type='glob') | dictsort() %} {% set hostname = minionid.split('_')[0] %} {% set node_type = minionid.split('_')[1] %} @@ -24,10 +22,10 @@ node_data: {% for node_type, host_values in node_types.items() %} - {{node_type}}: {% for hostname, details in host_values.items() %} - {{hostname}}: - ip: {{details.ip}} - alive: {{ details.alive }} + {{hostname}}: + ip: {{details.ip}} + alive: {{ details.alive }} + role: {{node_type}} {% endfor %} {% endfor %} diff --git a/pillar/top.sls b/pillar/top.sls index e0cac069f..96e0b7a53 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -10,6 +10,7 @@ base: - sensoroni.adv_sensoroni - telegraf.soc_telegraf - telegraf.adv_telegraf + - node_data.ips '* and not *_eval and not *_import': - logstash.nodes @@ -23,11 +24,15 @@ base: - logstash - logstash.manager - logstash.search + - logstash.soc_logstash + - logstash.adv_logstash - elasticsearch.index_templates '*_manager': - logstash - logstash.manager + - logstash.soc_logstash + - logstash.adv_logstash - elasticsearch.index_templates '*_manager or *_managersearch': @@ -51,6 +56,8 @@ base: - redis.adv_redis - influxdb.soc_influxdb - influxdb.adv_influxdb + - elasticsearch.soc_elasticsearch + - elasticsearch.adv_elasticsearch - backup.soc_backup - backup.adv_backup - minions.{{ grains.id }} @@ -76,6 +83,7 @@ base: - soc_global - kratos.soc_kratos - elasticsearch.soc_elasticsearch + - elasticsearch.adv_elasticsearch - manager.soc_manager - soc.soc_soc - kratos.soc_kratos @@ -94,6 +102,7 @@ base: - logstash.manager - logstash.search - logstash.soc_logstash + - logstash.adv_logstash - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth @@ -111,6 +120,7 @@ base: - influxdb.soc_influxdb - influxdb.adv_influxdb - elasticsearch.soc_elasticsearch + - elasticsearch.adv_elasticsearch - manager.soc_manager - soc.soc_soc - backup.soc_backup @@ -134,6 +144,8 @@ base: '*_searchnode': - logstash - logstash.search + - logstash.soc_logstash + - logstash.adv_logstash - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth @@ -148,6 +160,8 @@ base: '*_receiver': - logstash - logstash.receiver + - logstash.soc_logstash + - logstash.adv_logstash {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} @@ -169,6 +183,7 @@ base: {% endif %} - kratos.soc_kratos - elasticsearch.soc_elasticsearch + - elasticsearch.adv_elasticsearch - manager.soc_manager - soc.soc_soc - soc_global diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index c502c7b3e..abeb0004d 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -107,7 +107,8 @@ 'zeek', 'schedule', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'elastic-fleet' ], 'so-manager': [ 'salt.master', diff --git a/salt/common/init.sls b/salt/common/init.sls index a03eaacde..ea943ce4b 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -218,6 +218,11 @@ sostatus_check_cron: - month: '*' - dayweek: '*' +remove_post_setup_cron: + cron.absent: + - name: 'salt-call state.highstate' + - identifier: post_setup_cron + {% if GLOBALS.role not in ['eval', 'manager', 'managersearch', 'standalone'] %} soversionfile: diff --git a/salt/common/packages.sls b/salt/common/packages.sls index e52e8174f..0bb36d07f 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -1,6 +1,33 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} -{% if GLOBALS.os != 'CentOS' %} +{% if GLOBALS.os == 'Ubuntu' %} +commonpkgs: + pkg.installed: + - skip_suggestions: True + - pkgs: + - chrony + - apache2-utils + - wget + - ntpdate + - jq + - python3-docker + - curl + - ca-certificates + - software-properties-common + - apt-transport-https + - openssl + - netcat + - python3-mysqldb + - sqlite3 + - libssl-dev + - python3-dateutil + - python3-m2crypto + - python3-mysqldb + - python3-packaging + - python3-lxml + - git + - vim +{% elif GLOBALS.os == 'Rocky' %} commonpkgs: pkg.installed: - skip_suggestions: True diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete index 0564ad3c6..fe21daae0 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete +++ b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete @@ -1,4 +1,9 @@ #/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + . /usr/sbin/so-common POLICY_ID=$1 diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list index 8a9638922..479fdc72d 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list +++ b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list @@ -1,4 +1,9 @@ #/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + . /usr/sbin/so-common # Let's snag a cookie from Kibana diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view index 6ab6474b0..5f267220b 100644 --- a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view +++ b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view @@ -1,4 +1,9 @@ #/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + . /usr/sbin/so-common POLICY_ID=$1 diff --git a/salt/common/tools/sbin/so-elastic-fleet-data-streams-list b/salt/common/tools/sbin/so-elastic-fleet-data-streams-list index 9348428e8..238fc8ada 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-data-streams-list +++ b/salt/common/tools/sbin/so-elastic-fleet-data-streams-list @@ -1,4 +1,9 @@ #/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + . /usr/sbin/so-common # Let's snag a cookie from Kibana diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete new file mode 100644 index 000000000..4596774b4 --- /dev/null +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete @@ -0,0 +1,23 @@ +#/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +POLICY_ID=$1 + +# Let's snag a cookie from Kibana +SESSIONCOOKIE=$(curl -q -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') + +# Get integration policies relative to agent policy +INTEGRATION_POLICY_IDS=$(curl -q -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$POLICY_ID" | jq -r '.item.package_policies[].id') + +for i in $INTEGRATION_POLICY_IDS; do + # Delete integration policies + echo "Deleting integration policy: $i..." + curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d"{\"packagePolicyIds\": [\"$i\"], \"force\":true}"; + echo + echo +done diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete index b66b7eca8..a8601028c 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete @@ -1,4 +1,9 @@ #/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + . /usr/sbin/so-common POLICY_ID=$1 diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list index c73e84174..07d2822fb 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list @@ -1,4 +1,9 @@ #/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + . /usr/sbin/so-common # Let's snag a cookie from Kibana diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index 614fc81b1..f409d9a17 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -1,4 +1,8 @@ #/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. . /usr/sbin/so-common @@ -17,68 +21,76 @@ echo echo "Setting up default Security Onion package policies for Elastic Agent..." # Set up Suricata logs +echo echo "Setting up Suricata package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "suricata-logs", "name": "suricata-logs", "description": "Suricata integration", "namespace": "suricata_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/suricata/eve*.json" ], "data_stream.dataset": "alert", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata", "custom": "pipeline: suricata.common" }}}}}}' - -# Import - Suricata -echo "Settings up Suricata import package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-suricata-logs", "name": "import-suricata-logs", "description": "Import Suricata logs", "namespace": "import_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/suricata/eve*.json"], "data_stream.dataset": "alert", "tags": ["import"], "processors": "- add_fields:\n target: event\n fields:\n category: file\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"", "custom": "pipeline: suricata.common" } } } } } }' - -# Zeek logs -{%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} - {%- if ZEEKVER != 'SURICATA' %} - {% import_yaml 'filebeat/defaults.yaml' as FBD with context %} - {% set FBCONFIG = salt['pillar.get']('filebeat:zeek_logs_enabled', default=FBD.filebeat, merge=True) %} - {%- for LOGNAME in FBCONFIG.zeek_logs_enabled %} - -# Set up Zeek {{ LOGNAME }} logs -echo "Setting up Zeek {{ LOGNAME }} package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "zeek-{{ LOGNAME }}", "description": "Zeek {{ LOGNAME }} logs", "namespace": "zeek_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/zeek/logs/current/{{ LOGNAME }}.log" ], "data_stream.dataset": "{{ LOGNAME }}", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: zeek", "custom": "pipeline: zeek.{{ LOGNAME }}" }}}}}}' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "suricata-logs", "name": "suricata-logs", "description": "Suricata integration", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/suricata/eve*.json" ], "data_stream.dataset": "suricata", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata", "custom": "pipeline: suricata.common" }}}}}}' echo -echo "Setting up Zeek import {{ LOGNAME }} package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-zeek-{{ LOGNAME }}-logs", "name": "import-zeek-{{ LOGNAME }}-logs", "description": "Import Zeek {{ LOGNAME }} logs", "namespace": "import_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/zeek/logs/{{ LOGNAME }}.log"], "data_stream.dataset": "{{ LOGNAME }}", "tags": ["import"], "processors": "- add_fields:\n target: event\n fields:\n category: file\n module: suricata\n imported: true", "custom": "pipeline: zeek.{{ LOGNAME }}" } } } } } }' +# Set up Zeek logs +echo +echo "Setting up Zeek package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "zeek-logs", "description": "Zeek logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/zeek/logs/current/*.log"], "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", "custom": "prospector.scanner.exclude_files: [\"(broker | capture_loss | loaded_scripts | packet_filter | stats | stderr | stdout).log$\"]\n" } } } } } }' +echo - {%- endfor %} - {%- endif %} -{%- endif %} +# Import - Suricata +echo +echo "Settings up Suricata import package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-suricata-logs", "name": "import-suricata-logs", "description": "Import Suricata logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/suricata/eve*.json"], "data_stream.dataset": "import", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: file\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"", "custom": "pipeline: suricata.common" } } } } } }' +echo +# Set Import - Zeek logs +echo +echo "Setting up Zeek Import package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "import-zeek-logs", "description": "Zeek Import logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/zeek/logs/*.log"], "data_stream.dataset": "import", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"import.file\").slice(0,-4);\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n imported: true\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"" } } } } } }' # Strelka logs echo "Setting up Strelka package policy" -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "strelka-logs", "name": "strelka-logs", "description": "Strelka logs", "namespace": "strelka_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], "data_stream.dataset": "file", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: file\n module: strelka", "custom": "pipeline: strelka.file" }}}}}}' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "strelka-logs", "name": "strelka-logs", "description": "Strelka logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], "data_stream.dataset": "file", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: file\n module: strelka", "custom": "pipeline: strelka.file" }}}}}}' +echo # Syslog TCP Port 514 +echo echo "Setting up Syslog TCP package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "tcp", "version": "1.5.0" }, "id": "syslog-tcp-514", "name": "syslog-tcp-514", "description": "Syslog Over TCP Port 514", "namespace": "syslog_so", "inputs": { "tcp-tcp": { "enabled": true, "streams": { "tcp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "processors": "- add_fields:\n target: event\n fields:\n module: syslog", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "tcp", "version": "1.5.0" }, "id": "syslog-tcp-514", "name": "syslog-tcp-514", "description": "Syslog Over TCP Port 514", "namespace": "so", "inputs": { "tcp-tcp": { "enabled": true, "streams": { "tcp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "processors": "- add_fields:\n target: event\n fields:\n module: syslog", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' +echo # Syslog UDP Port 514 +echo echo "Setting up Syslog UDP package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "udp", "version": "1.5.0" }, "id": "syslog-udp-514", "name": "syslog-udp-514", "description": "Syslog over UDP Port 514", "namespace": "syslog_so", "inputs": { "udp-udp": { "enabled": true, "streams": { "udp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "max_message_size": "10KiB", "keep_null": false, "processors": "- add_fields:\n target: event\n fields: \n module: syslog\n", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "udp", "version": "1.5.0" }, "id": "syslog-udp-514", "name": "syslog-udp-514", "description": "Syslog over UDP Port 514", "namespace": "so", "inputs": { "udp-udp": { "enabled": true, "streams": { "udp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "max_message_size": "10KiB", "keep_null": false, "processors": "- add_fields:\n target: event\n fields: \n module: syslog\n", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' +echo # Kratos logs +echo echo "Setting up Kratos package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "kratos_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "access", "tags": [],"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- rename:\n fields:\n - from: \"audience\"\n to: \"event.dataset\"\n ignore_missing: true\n- add_fields:\n when:\n not: \n has_fields: ['event.dataset']\n target: ''\n fields:\n event.dataset: access", "custom": "pipeline: kratos" }}}}}}' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "kratos", "tags": [],"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- rename:\n fields:\n - from: \"audience\"\n to: \"event.dataset\"\n ignore_missing: true\n- add_fields:\n when:\n not: \n has_fields: ['event.dataset']\n target: ''\n fields:\n event.dataset: access", "custom": "pipeline: kratos" }}}}}}' +echo -# RITA Beacon logs -echo "Setting up RITA Beacon package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-beacon-logs", "name": "rita-beacon-logs", "description": "RITA Beacon logs", "namespace": "rita_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv" ], "data_stream.dataset": "beacon", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.beacon" }}}}}}' - -# RITA Connection Logs -echo "Setting up RITA connection package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-connection-logs", "name": "rita-connection-logs", "description": "RITA Connection logs", "namespace": "rita_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/long-connections.csv" ], "data_stream.dataset": "connections", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.connections" }}}}}}' - -# RITA DNS Logs -echo "Setting up RITA DNS package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-exploded-dns-logs", "name": "rita-exploded-dns-logs", "description": "RITA Exploded DNS logs", "namespace": "rita_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "exploded_dns", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.dns" }}}}}}' +# RITA Logs +#echo +#echo "Setting up RITA package policy..." +#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita\n- if:\n log.file.path: beacons.csv\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.beacon\n- if:\n regexp:\n log.file.path: \"*connections.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.connection\n- if:\n log.file.path: \"exploded-dns.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.dns" }}}}}}' +#echo # Elasticsearch logs +echo +echo "Seting up Elasticsearch package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "elasticsearch", "version": "1.0.0" }, "id": "elasticsearch-logs", "name": "elasticsearch-logs", "description": "Elasticsearch Logs", "namespace": "default", "inputs": { "elasticsearch-logfile": { "enabled": true, "streams": { "elasticsearch.audit": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_audit.json" ] } }, "elasticsearch.deprecation": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_deprecation.json" ] } }, "elasticsearch.gc": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/gc.log.[0-9]*", "/var/log/elasticsearch/gc.log" ] } }, "elasticsearch.server": { "enabled": true, "vars": { "paths": [ "/opt/so/log/elasticsearch/*.log" ] } }, "elasticsearch.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_index_search_slowlog.json", "/var/log/elasticsearch/*_index_indexing_slowlog.json" ] } } } }, "elasticsearch-elasticsearch/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9200" ], "scope": "node" }, "streams": { "elasticsearch.stack_monitoring.ccr": { "enabled": false }, "elasticsearch.stack_monitoring.cluster_stats": { "enabled": false }, "elasticsearch.stack_monitoring.enrich": { "enabled": false }, "elasticsearch.stack_monitoring.index": { "enabled": false }, "elasticsearch.stack_monitoring.index_recovery": { "enabled": false, "vars": { "active.only": true } }, "elasticsearch.stack_monitoring.index_summary": { "enabled": false }, "elasticsearch.stack_monitoring.ml_job": { "enabled": false }, "elasticsearch.stack_monitoring.node": { "enabled": false }, "elasticsearch.stack_monitoring.node_stats": { "enabled": false }, "elasticsearch.stack_monitoring.pending_tasks": { "enabled": false }, "elasticsearch.stack_monitoring.shard": { "enabled": false } } } } }' +echo # Logstash logs +echo +echo "Setting up Logstash package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "logstash", "version": "2.0.0" }, "id": "logstash-logs", "name": "logstash-logs", "description": "Logstash logs", "namespace": "default", "inputs": { "logstash-logfile": { "enabled": true, "streams": { "logstash.log": { "enabled": true, "vars": { "paths": [ "/opt/so/logs/logstash/logstash.log" ] } }, "logstash.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/logstash/logstash-slowlog-plain*.log", "/var/log/logstash/logstash-slowlog-json*.log" ] } } } }, "logstash-logstash/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9600" ], "period": "10s" }, "streams": { "logstash.stack_monitoring.node": { "enabled": false }, "logstash.stack_monitoring.node_stats": { "enabled": false } } } } }' +echo # Kibana logs +echo +echo "Setting up Kibana package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "kibana", "version": "2.0.0" }, "id": "kibana-logs", "name": "kibana-logs", "description": "Kibana logs", "namespace": "default", "inputs": { "kibana-logfile": { "enabled": true, "streams": { "kibana.audit": { "enabled": false, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } }, "kibana.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } } } }, "kibana-kibana/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:5601" ] }, "streams": { "kibana.stack_monitoring.cluster_actions": { "enabled": false }, "kibana.stack_monitoring.cluster_rules": { "enabled": false }, "kibana.stack_monitoring.node_actions": { "enabled": false }, "kibana.stack_monitoring.node_rules": { "enabled": false }, "kibana.stack_monitoring.stats": { "enabled": false }, "kibana.stack_monitoring.status": { "enabled": false } } } } }' +echo # Redis logs +echo +echo "Setting up Redis package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "redis", "version": "1.4.0" }, "id": "redis-logs", "name": "redis-logs", "description": "Redis logs", "namespace": "default", "inputs": { "redis-logfile": { "enabled": true, "streams": { "redis.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/redis/redis.log" ], "tags": [ "redis-log" ], "preserve_original_event": false } } } }, "redis-redis": { "enabled": false, "streams": { "redis.slowlog": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "password": "" } } } }, "redis-redis/metrics": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "idle_timeout": "20s", "maxconn": 10, "network": "tcp", "password": "" }, "streams": { "redis.info": { "enabled": false, "vars": { "period": "10s" } }, "redis.key": { "enabled": false, "vars": { "key.patterns": "- limit: 20\n pattern: '*'\n", "period": "10s" } }, "redis.keyspace": { "enabled": false, "vars": { "period": "10s" } } } } } }' +echo diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index 4e3d7639c..c945aeec3 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -19,10 +19,22 @@ printf "\n" curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220"]}' printf "\n\n" -# Create Logstash Output payload +# Configure certificates mkdir -p /opt/so/conf/elastic-fleet/certs cp /etc/ssl/certs/intca.crt /opt/so/conf/elastic-fleet/certs cp /etc/pki/elasticfleet* /opt/so/conf/elastic-fleet/certs + +{% if grains.role == 'so-import' %} +# Add SO-Manager Elasticsearch Ouput +ESCACRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/intca.crt) +JSON_STRING=$( jq -n \ + --arg ESCACRT "$ESCACRT" \ + '{"name":"so-manager_elasticsearch2","id":"so-manager_elasticsearch2","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' ) +curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +printf "\n\n" + +{% else %} +# Create Logstash Output payload LOGSTASHCRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/elasticfleet.crt) LOGSTASHKEY=$(openssl rsa -in /opt/so/conf/elastic-fleet/certs/elasticfleet.key) LOGSTASHCA=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/intca.crt) @@ -30,12 +42,13 @@ JSON_STRING=$( jq -n \ --arg LOGSTASHCRT "$LOGSTASHCRT" \ --arg LOGSTASHKEY "$LOGSTASHKEY" \ --arg LOGSTASHCA "$LOGSTASHCA" \ - '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}' + '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}' ) # Add SO-Manager Logstash Ouput curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" +{%- endif %} # Add Elastic Fleet Integrations diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 3851d8b4a..945d3f4ed 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -36,7 +36,9 @@ container_list() { "so-steno" "so-suricata" "so-telegraf" - "so-zeek" + "so-zeek" + "so-elastic-agent" + "so-elastic-agent-builder" ) elif [ $MANAGERCHECK != 'so-helix' ]; then TRUSTED_CONTAINERS=( diff --git a/salt/common/tools/sbin/so-minion b/salt/common/tools/sbin/so-minion index f0b0c4ffe..cce5e2a64 100755 --- a/salt/common/tools/sbin/so-minion +++ b/salt/common/tools/sbin/so-minion @@ -131,13 +131,24 @@ function add_fleet_to_minion() { # Write out settings to minion file printf '%s\n'\ - "logstash_settings:"\ - " lsheap: '500m'"\ "fleet-server:"\ " ES-Token: '$ESTOKEN'"\ " " >> $PILLARFILE } + +function add_logstash_to_minion() { + # Create the logstash advanced pillar + printf '%s\n'\ + "logstash_settings:"\ + " ls_host: '$LSHOSTNAME'"\ + " ls_pipeline_batch_size: 125"\ + " ls_input_threads: 1"\ + " lsheap: $LSHEAP"\ + " ls_pipeline_workers: $CPUCORES"\ + " " >> $PILLARFILE +} + # Analyst Workstation function add_analyst_to_minion() { printf '%s\n'\ @@ -186,11 +197,13 @@ function add_sensor_to_minion() { function createEVAL() { add_elastic_to_minion + add_logstash_to_minion add_sensor_to_minion } function createFLEET() { add_fleet_to_minion + add_logstash_to_minion } function createIDHNODE() { @@ -199,20 +212,24 @@ function createIDHNODE() { function createIMPORT() { add_elastic_to_minion + add_logstash_to_minion add_sensor_to_minion } function createHEAVYNODE() { add_elastic_to_minion + add_logstash_to_minion add_sensor_to_minion } -function createMASTER() { +function createMANAGER() { add_elastic_to_minion + add_logstash_to_minion } -function createMASTERSEARCH() { +function createMANAGERSEARCH() { add_elastic_to_minion + add_logstash_to_minion } function createSENSOR() { @@ -221,10 +238,12 @@ function createSENSOR() { function createSEARCHNODE() { add_elastic_to_minion + add_logstash_to_minion } function createSTANDALONE() { add_elastic_to_minion + add_logstash_to_minion add_sensor_to_minion } diff --git a/salt/docker/init.sls b/salt/docker/init.sls index f8f89e058..405a75938 100644 --- a/salt/docker/init.sls +++ b/salt/docker/init.sls @@ -4,12 +4,25 @@ # Elastic License 2.0. {% from 'docker/docker.map.jinja' import DOCKER %} +{% from 'vars/globals.map.jinja' import GLOBALS %} + dockergroup: group.present: - name: docker - gid: 920 +{% if GLOBALS.os == 'Ubuntu' %} +dockerheldpackages: + pkg.installed: + - pkgs: + - containerd.io: 1.4.9-1 + - docker-ce: 5:20.10.8~3-0~ubuntu-focal + - docker-ce-cli: 5:20.10.5~3-0~ubuntu-focal + - docker-ce-rootless-extras: 5:20.10.5~3-0~ubuntu-focal + - hold: True + - update_holds: True +{% else %} dockerheldpackages: pkg.installed: - pkgs: @@ -19,6 +32,7 @@ dockerheldpackages: - docker-ce-rootless-extras: 20.10.5-3.el7 - hold: True - update_holds: True +{% endif %} #disable docker from managing iptables iptables_disabled: diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index ff96d139b..3d1182255 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1165,7 +1165,7 @@ elasticsearch: index_template: data_stream: {} index_patterns: - - logs-*_so + - logs-*-so* template: mappings: dynamic_templates: @@ -1418,7 +1418,7 @@ elasticsearch: index_sorting: False index_template: index_patterns: - - so-elasticsearch* + - logs-elasticsearch-default* template: mappings: dynamic_templates: @@ -2171,7 +2171,7 @@ elasticsearch: index_template: data_stream: {} index_patterns: - - logs-*-suricata_so + - logs-suricata-so* template: mappings: dynamic_templates: @@ -2339,7 +2339,7 @@ elasticsearch: index_template: data_stream: {} index_patterns: - - logs-*-import_so + - logs-import-so* template: mappings: dynamic_templates: @@ -2677,8 +2677,11 @@ elasticsearch: delete: 365 index_sorting: False index_template: + data_stream: + hidden: false + allow_custom_routing: false index_patterns: - - logs-*-kratos_so + - logs-kratos-so* template: mappings: dynamic_templates: @@ -2755,7 +2758,7 @@ elasticsearch: index_sorting: False index_template: index_patterns: - - so-logstash* + - logs-logstash-default* template: mappings: dynamic_templates: @@ -3676,7 +3679,7 @@ elasticsearch: index_sorting: False index_template: index_patterns: - - so-redis* + - logs-redis-default* template: mappings: dynamic_templates: @@ -4178,7 +4181,7 @@ elasticsearch: index_template: data_stream: {} index_patterns: - - logs-*-strelka_so + - logs-strelka-so* template: mappings: dynamic_templates: @@ -4263,7 +4266,7 @@ elasticsearch: index_sorting: False index_template: index_patterns: - - so-syslog* + - logs-syslog-so* template: mappings: dynamic_templates: @@ -4432,7 +4435,7 @@ elasticsearch: index_template: data_stream: {} index_patterns: - - logs-*-zeek_so + - logs-zeek-so* template: mappings: dynamic_templates: diff --git a/salt/elasticsearch/files/ingest/kratos b/salt/elasticsearch/files/ingest/kratos index c55ebd60a..f8dcf53e0 100644 --- a/salt/elasticsearch/files/ingest/kratos +++ b/salt/elasticsearch/files/ingest/kratos @@ -1,6 +1,7 @@ { "description" : "kratos", "processors" : [ + { "set": { "field": "event.dataset", "value": "access" } }, { "pipeline": { "name": "common" } } ] } diff --git a/salt/elasticsearch/files/ingest/rita.beacon b/salt/elasticsearch/files/ingest/rita.beacon index 2c6138262..81bd34e9d 100644 --- a/salt/elasticsearch/files/ingest/rita.beacon +++ b/salt/elasticsearch/files/ingest/rita.beacon @@ -3,8 +3,8 @@ "processors": [ { "set": { - "field": "_index", - "value": "so-rita", + "field": "event.dataset", + "value": "beacon", "override": true } }, diff --git a/salt/elasticsearch/files/ingest/rita.connection b/salt/elasticsearch/files/ingest/rita.connection index 58cc921f4..2c0e33c87 100644 --- a/salt/elasticsearch/files/ingest/rita.connection +++ b/salt/elasticsearch/files/ingest/rita.connection @@ -3,8 +3,8 @@ "processors": [ { "set": { - "field": "_index", - "value": "so-rita", + "field": "event.dataset", + "value": "connection", "override": true } }, diff --git a/salt/elasticsearch/files/ingest/rita.dns b/salt/elasticsearch/files/ingest/rita.dns index 7583bc320..837571b70 100644 --- a/salt/elasticsearch/files/ingest/rita.dns +++ b/salt/elasticsearch/files/ingest/rita.dns @@ -3,8 +3,8 @@ "processors": [ { "set": { - "field": "_index", - "value": "so-rita", + "field": "event.dataset", + "value": "dns", "override": true } }, diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common index a8be6d46b..6e956d196 100644 --- a/salt/elasticsearch/files/ingest/suricata.common +++ b/salt/elasticsearch/files/ingest/suricata.common @@ -9,12 +9,11 @@ { "rename":{ "field": "message2.dest_ip", "target_field": "destination.ip", "ignore_failure": true } }, { "rename":{ "field": "message2.dest_port", "target_field": "destination.port", "ignore_failure": true } }, { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_missing": true } }, - { "remove":{ "field": "dataset", "ignore_failure": true } }, - { "rename":{ "field": "message2.event_type", "target_field": "dataset", "ignore_failure": true } }, + { "set": { "field": "event.dataset", "value": "{{ message2.event_type }}" } }, { "set": { "field": "observer.name", "value": "{{agent.name}}" } }, { "set": { "field": "event.ingested", "value": "{{@timestamp}}" } }, { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } }, { "remove":{ "field": "agent", "ignore_failure": true } }, - { "pipeline": { "if": "ctx?.dataset != null", "name": "suricata.{{dataset}}" } } + { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } } ] -} \ No newline at end of file +} diff --git a/salt/elasticsearch/files/ingest/zeek.bacnet b/salt/elasticsearch/files/ingest/zeek.bacnet index a96a05b56..08e89d7a3 100644 --- a/salt/elasticsearch/files/ingest/zeek.bacnet +++ b/salt/elasticsearch/files/ingest/zeek.bacnet @@ -1,6 +1,7 @@ { "description" : "zeek.bacnet", "processors" : [ + { "set": { "field": "event.dataset", "value": "bacnet" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.is_orig", "target_field": "bacnet.is_orig", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bacnet_discovery b/salt/elasticsearch/files/ingest/zeek.bacnet_discovery index c51cbf5c1..f848e683a 100644 --- a/salt/elasticsearch/files/ingest/zeek.bacnet_discovery +++ b/salt/elasticsearch/files/ingest/zeek.bacnet_discovery @@ -1,6 +1,7 @@ { "description" : "zeek.bacnet_discovery", "processors" : [ + { "set": { "field": "event.dataset", "value": "bacnet_discovery" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.is_orig", "target_field": "bacnet.is_orig", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bacnet_property b/salt/elasticsearch/files/ingest/zeek.bacnet_property index d04c4c327..80e23652f 100644 --- a/salt/elasticsearch/files/ingest/zeek.bacnet_property +++ b/salt/elasticsearch/files/ingest/zeek.bacnet_property @@ -1,6 +1,7 @@ { "description" : "zeek.bacnet_property", "processors" : [ + { "set": { "field": "event.dataset", "value": "bacnet_property" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.is_orig", "target_field": "bacnet.is_orig", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bsap_ip_header b/salt/elasticsearch/files/ingest/zeek.bsap_ip_header index b92d7f233..965d642df 100644 --- a/salt/elasticsearch/files/ingest/zeek.bsap_ip_header +++ b/salt/elasticsearch/files/ingest/zeek.bsap_ip_header @@ -1,6 +1,7 @@ { "description" : "zeek.bsap_ip_header", "processors" : [ + { "set": { "field": "event.dataset", "value": "bsap_ip_header" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.num_msg", "target_field": "bsap.number.messages", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bsap_ip_rdb b/salt/elasticsearch/files/ingest/zeek.bsap_ip_rdb index f5ebd3a0a..aade31c21 100644 --- a/salt/elasticsearch/files/ingest/zeek.bsap_ip_rdb +++ b/salt/elasticsearch/files/ingest/zeek.bsap_ip_rdb @@ -1,6 +1,7 @@ { "description" : "zeek.bsap_ip_rdb", "processors" : [ + { "set": { "field": "event.dataset", "value": "bsap_ip_rdb" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.header_size", "target_field": "bsap.header.length", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bsap_ip_unknown b/salt/elasticsearch/files/ingest/zeek.bsap_ip_unknown index 15d9a8fd9..d561c90b0 100644 --- a/salt/elasticsearch/files/ingest/zeek.bsap_ip_unknown +++ b/salt/elasticsearch/files/ingest/zeek.bsap_ip_unknown @@ -1,6 +1,7 @@ { "description" : "zeek.bsap_ip_unknown", "processors" : [ + { "set": { "field": "event.dataset", "value": "bsap_ip_unknown" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.data", "target_field": "bsap.ip.unknown.data", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bsap_serial_header b/salt/elasticsearch/files/ingest/zeek.bsap_serial_header index b4b499ff1..2cfcd7ecf 100644 --- a/salt/elasticsearch/files/ingest/zeek.bsap_serial_header +++ b/salt/elasticsearch/files/ingest/zeek.bsap_serial_header @@ -1,6 +1,7 @@ { "description" : "zeek.bsap_serial_header", "processors" : [ + { "set": { "field": "event.dataset", "value": "bsap_serial_header" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.ser", "target_field": "bsap.message.serial_number", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb b/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb index f45d8c0cd..8e7f07802 100644 --- a/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb +++ b/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb @@ -1,6 +1,7 @@ { "description" : "zeek.bsap_serial_rdb", "processors" : [ + { "set": { "field": "event.dataset", "value": "bsap_serial_rdb" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.func_code", "target_field": "bsap.rdb.function", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb_ext b/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb_ext index 2ca85ad78..110095b59 100644 --- a/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb_ext +++ b/salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb_ext @@ -1,6 +1,7 @@ { "description" : "zeek.bsap_serial_rdb_ext", "processors" : [ + { "set": { "field": "event.dataset", "value": "bsap_serial_rdb_ext" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.dfun", "target_field": "bsap.destination.function", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.bsap_serial_unknown b/salt/elasticsearch/files/ingest/zeek.bsap_serial_unknown index ea0c5e471..bb30c5940 100644 --- a/salt/elasticsearch/files/ingest/zeek.bsap_serial_unknown +++ b/salt/elasticsearch/files/ingest/zeek.bsap_serial_unknown @@ -1,6 +1,7 @@ { "description" : "zeek.bsap_serial_unknown", "processors" : [ + { "set": { "field": "event.dataset", "value": "bsap_serial_unknown" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.data", "target_field": "bsap.serial.unknown.data", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.cip b/salt/elasticsearch/files/ingest/zeek.cip index 5182a7037..a704c2b28 100644 --- a/salt/elasticsearch/files/ingest/zeek.cip +++ b/salt/elasticsearch/files/ingest/zeek.cip @@ -1,6 +1,7 @@ { "description" : "zeek.cip", "processors" : [ + { "set": { "field": "event.dataset", "value": "cip" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.is_orig", "target_field": "cip.is_orig", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.cip_identity b/salt/elasticsearch/files/ingest/zeek.cip_identity index a3522f86a..e36678f66 100644 --- a/salt/elasticsearch/files/ingest/zeek.cip_identity +++ b/salt/elasticsearch/files/ingest/zeek.cip_identity @@ -1,6 +1,7 @@ { "description" : "zeek.cip_identity", "processors" : [ + { "set": { "field": "event.dataset", "value": "cip_identity" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.encapsulation_version", "target_field": "cip.encapsulation.version", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.cip_io b/salt/elasticsearch/files/ingest/zeek.cip_io index 68c376b05..f41c3f431 100644 --- a/salt/elasticsearch/files/ingest/zeek.cip_io +++ b/salt/elasticsearch/files/ingest/zeek.cip_io @@ -1,6 +1,7 @@ { "description" : "zeek.cip_io", "processors" : [ + { "set": { "field": "event.dataset", "value": "cip_io" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.is_orig", "target_field": "cip.is_orig", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.conn b/salt/elasticsearch/files/ingest/zeek.conn index 14689f782..73ce29c1d 100644 --- a/salt/elasticsearch/files/ingest/zeek.conn +++ b/salt/elasticsearch/files/ingest/zeek.conn @@ -1,6 +1,7 @@ { "description" : "zeek.conn", "processors" : [ + { "set": { "field": "event.dataset", "value": "conn" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.cotp b/salt/elasticsearch/files/ingest/zeek.cotp index fb4b090cd..f5abd87f6 100644 --- a/salt/elasticsearch/files/ingest/zeek.cotp +++ b/salt/elasticsearch/files/ingest/zeek.cotp @@ -1,6 +1,7 @@ { "description" : "zeek.cotp", "processors" : [ + { "set": { "field": "event.dataset", "value": "cotp" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.pdu_code", "target_field": "cotp.pdu.code", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.dce_rpc b/salt/elasticsearch/files/ingest/zeek.dce_rpc index 247946073..0f6ae6d67 100644 --- a/salt/elasticsearch/files/ingest/zeek.dce_rpc +++ b/salt/elasticsearch/files/ingest/zeek.dce_rpc @@ -1,6 +1,7 @@ { "description" : "zeek.dce_rpc", "processors" : [ + { "set": { "field": "event.dataset", "value": "dce_rpc" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.rtt", "target_field": "event.duration", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.dhcp b/salt/elasticsearch/files/ingest/zeek.dhcp index f3a551edd..d2d151847 100644 --- a/salt/elasticsearch/files/ingest/zeek.dhcp +++ b/salt/elasticsearch/files/ingest/zeek.dhcp @@ -1,6 +1,7 @@ { "description" : "zeek.dhcp", "processors" : [ + { "set": { "field": "event.dataset", "value": "dhcp" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.uids", "target_field": "log.id.uids", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.dnp3 b/salt/elasticsearch/files/ingest/zeek.dnp3 index 6618289f8..e8011463b 100644 --- a/salt/elasticsearch/files/ingest/zeek.dnp3 +++ b/salt/elasticsearch/files/ingest/zeek.dnp3 @@ -1,6 +1,7 @@ { "description" : "zeek.dnp3", "processors" : [ + { "set": { "field": "event.dataset", "value": "dnp3" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.fc_request", "target_field": "dnp3.fc_request", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.dnp3_control b/salt/elasticsearch/files/ingest/zeek.dnp3_control index 0c465b5d2..e6a3a106c 100644 --- a/salt/elasticsearch/files/ingest/zeek.dnp3_control +++ b/salt/elasticsearch/files/ingest/zeek.dnp3_control @@ -1,6 +1,7 @@ { "description" : "zeek.dnp3_control", "processors" : [ + { "set": { "field": "event.dataset", "value": "dnp3_control" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.block_type", "target_field": "dnp3.block_type", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.dnp3_objects b/salt/elasticsearch/files/ingest/zeek.dnp3_objects index c78ae9e1f..bdce7e8d1 100644 --- a/salt/elasticsearch/files/ingest/zeek.dnp3_objects +++ b/salt/elasticsearch/files/ingest/zeek.dnp3_objects @@ -1,6 +1,7 @@ { "description" : "zeek.dnp3_objects", "processors" : [ + { "set": { "field": "event.dataset", "value": "dnp3_objects" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.function_code", "target_field": "dnp3.function_code", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.dns b/salt/elasticsearch/files/ingest/zeek.dns index 9d7f7e93b..5f39b8017 100644 --- a/salt/elasticsearch/files/ingest/zeek.dns +++ b/salt/elasticsearch/files/ingest/zeek.dns @@ -1,6 +1,7 @@ { "description" : "zeek.dns", "processors" : [ + { "set": { "field": "event.dataset", "value": "dns" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.dpd b/salt/elasticsearch/files/ingest/zeek.dpd index 59ceed368..2f76c5ecb 100644 --- a/salt/elasticsearch/files/ingest/zeek.dpd +++ b/salt/elasticsearch/files/ingest/zeek.dpd @@ -1,6 +1,7 @@ { "description" : "zeek.dpd", "processors" : [ + { "set": { "field": "event.dataset", "value": "dpd" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_aoe_info b/salt/elasticsearch/files/ingest/zeek.ecat_aoe_info index c5f9b9dc3..e9c056666 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_aoe_info +++ b/salt/elasticsearch/files/ingest/zeek.ecat_aoe_info @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_aoe_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_aoe_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.targetid", "target_field": "destination.mac", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_arp_info b/salt/elasticsearch/files/ingest/zeek.ecat_arp_info index cbc3676ab..750217dd3 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_arp_info +++ b/salt/elasticsearch/files/ingest/zeek.ecat_arp_info @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_arp_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_arp_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.arp_type", "target_field": "ecat.arp.type", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_coe_info b/salt/elasticsearch/files/ingest/zeek.ecat_coe_info index e425e3173..8571ece05 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_coe_info +++ b/salt/elasticsearch/files/ingest/zeek.ecat_coe_info @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_coe_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_coe_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.number", "target_field": "ecat.message.number", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_dev_info b/salt/elasticsearch/files/ingest/zeek.ecat_dev_info index d01289e9a..165c04134 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_dev_info +++ b/salt/elasticsearch/files/ingest/zeek.ecat_dev_info @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_dev_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_dev_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.slave_id", "target_field": "ecat.slave.address", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_foe_info b/salt/elasticsearch/files/ingest/zeek.ecat_foe_info index 6655f2cb7..810435827 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_foe_info +++ b/salt/elasticsearch/files/ingest/zeek.ecat_foe_info @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_foe_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_foe_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcode", "target_field": "ecat.operation.code", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_log_address b/salt/elasticsearch/files/ingest/zeek.ecat_log_address index ad0ee161f..c3f15ca29 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_log_address +++ b/salt/elasticsearch/files/ingest/zeek.ecat_log_address @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_log_address", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_log_address" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.srcmac", "target_field": "source.mac", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_registers b/salt/elasticsearch/files/ingest/zeek.ecat_registers index d0a11ba83..363a71918 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_registers +++ b/salt/elasticsearch/files/ingest/zeek.ecat_registers @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_registers", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_registers" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.srcmac", "target_field": "source.mac", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ecat_soe_info b/salt/elasticsearch/files/ingest/zeek.ecat_soe_info index bddc40efa..5785348bb 100644 --- a/salt/elasticsearch/files/ingest/zeek.ecat_soe_info +++ b/salt/elasticsearch/files/ingest/zeek.ecat_soe_info @@ -1,6 +1,7 @@ { "description" : "zeek.ecat_soe_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "ecat_soe_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcode", "target_field": "ecat.operation.code", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.enip b/salt/elasticsearch/files/ingest/zeek.enip index fee3b058b..443a72f9a 100644 --- a/salt/elasticsearch/files/ingest/zeek.enip +++ b/salt/elasticsearch/files/ingest/zeek.enip @@ -1,6 +1,7 @@ { "description" : "zeek.enip", "processors" : [ + { "set": { "field": "event.dataset", "value": "enip" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.is_orig", "target_field": "enip.is_orig", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.files b/salt/elasticsearch/files/ingest/zeek.files index c5fefa85f..105bbd326 100644 --- a/salt/elasticsearch/files/ingest/zeek.files +++ b/salt/elasticsearch/files/ingest/zeek.files @@ -1,6 +1,7 @@ { "description" : "zeek.files", "processors" : [ + { "set": { "field": "event.dataset", "value": "files" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.fuid", "target_field": "log.id.fuid", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ftp b/salt/elasticsearch/files/ingest/zeek.ftp index 45aa76055..3767c7b10 100644 --- a/salt/elasticsearch/files/ingest/zeek.ftp +++ b/salt/elasticsearch/files/ingest/zeek.ftp @@ -1,6 +1,7 @@ { "description" : "zeek.ftp", "processors" : [ + { "set": { "field": "event.dataset", "value": "ftp" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.user", "target_field": "ftp.user", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.http b/salt/elasticsearch/files/ingest/zeek.http index 3368e45e1..2224da2f7 100644 --- a/salt/elasticsearch/files/ingest/zeek.http +++ b/salt/elasticsearch/files/ingest/zeek.http @@ -1,6 +1,7 @@ { "description" : "zeek.http", "processors" : [ + { "set": { "field": "event.dataset", "value": "http" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.trans_depth", "target_field": "http.trans_depth", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.intel b/salt/elasticsearch/files/ingest/zeek.intel index 8be25c9ef..f3553ebd3 100644 --- a/salt/elasticsearch/files/ingest/zeek.intel +++ b/salt/elasticsearch/files/ingest/zeek.intel @@ -1,6 +1,7 @@ { "description" : "zeek.intel", "processors" : [ + { "set": { "field": "event.dataset", "value": "intel" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "dot_expander": { "field": "seen.indicator", "path": "message2", "ignore_failure": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.irc b/salt/elasticsearch/files/ingest/zeek.irc index 3d40d7f80..e21f36553 100644 --- a/salt/elasticsearch/files/ingest/zeek.irc +++ b/salt/elasticsearch/files/ingest/zeek.irc @@ -1,6 +1,7 @@ { "description" : "zeek.irc", "processors" : [ + { "set": { "field": "event.dataset", "value": "irc" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.nick", "target_field": "irc.nickname", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.kerberos b/salt/elasticsearch/files/ingest/zeek.kerberos index 0b09ebf61..f8aec42ec 100644 --- a/salt/elasticsearch/files/ingest/zeek.kerberos +++ b/salt/elasticsearch/files/ingest/zeek.kerberos @@ -1,6 +1,7 @@ { "description" : "zeek.kerberos", "processors" : [ + { "set": { "field": "event.dataset", "value": "kerberos" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.request_type", "target_field": "kerberos.request_type", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.modbus b/salt/elasticsearch/files/ingest/zeek.modbus index a9dd91430..92d012862 100644 --- a/salt/elasticsearch/files/ingest/zeek.modbus +++ b/salt/elasticsearch/files/ingest/zeek.modbus @@ -1,6 +1,7 @@ { "description" : "zeek.modbus", "processors" : [ + { "set": { "field": "event.dataset", "value": "modbus" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.func", "target_field": "modbus.function", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.modbus_detailed b/salt/elasticsearch/files/ingest/zeek.modbus_detailed index 635566c6a..f5bdf8200 100644 --- a/salt/elasticsearch/files/ingest/zeek.modbus_detailed +++ b/salt/elasticsearch/files/ingest/zeek.modbus_detailed @@ -1,6 +1,7 @@ { "description" : "zeek.modbus_detailed", "processors" : [ + { "set": { "field": "event.dataset", "value": "modbus_detailed" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.unit_id", "target_field": "modbus.unit_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.modbus_mask_write_register b/salt/elasticsearch/files/ingest/zeek.modbus_mask_write_register index d548fe615..9e4a0b21b 100644 --- a/salt/elasticsearch/files/ingest/zeek.modbus_mask_write_register +++ b/salt/elasticsearch/files/ingest/zeek.modbus_mask_write_register @@ -1,6 +1,7 @@ { "description" : "zeek.modbus_mask_write_register", "processors" : [ + { "set": { "field": "event.dataset", "value": "modbus_mask_write_register" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.unit_id", "target_field": "modbus.unit_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.modbus_read_write_multiple_registers b/salt/elasticsearch/files/ingest/zeek.modbus_read_write_multiple_registers index 234faa34f..2c7bf04b4 100644 --- a/salt/elasticsearch/files/ingest/zeek.modbus_read_write_multiple_registers +++ b/salt/elasticsearch/files/ingest/zeek.modbus_read_write_multiple_registers @@ -1,6 +1,7 @@ { "description" : "zeek.read_write_multiple_registers", "processors" : [ + { "set": { "field": "event.dataset", "value": "modbus_read_write_multiple_registers" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.unit_id", "target_field": "modbus.unit_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.mysql b/salt/elasticsearch/files/ingest/zeek.mysql index e950d5f64..3e57e2a29 100644 --- a/salt/elasticsearch/files/ingest/zeek.mysql +++ b/salt/elasticsearch/files/ingest/zeek.mysql @@ -1,6 +1,7 @@ { "description" : "zeek.mysql", "processors" : [ + { "set": { "field": "event.dataset", "value": "mysql" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.cmd", "target_field": "mysql.command", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.notice b/salt/elasticsearch/files/ingest/zeek.notice index b662393f6..8e086ad05 100644 --- a/salt/elasticsearch/files/ingest/zeek.notice +++ b/salt/elasticsearch/files/ingest/zeek.notice @@ -1,6 +1,7 @@ { "description" : "zeek.notice", "processors" : [ + { "set": { "field": "event.dataset", "value": "notice" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.fuid", "target_field": "log.id.fuid", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ntlm b/salt/elasticsearch/files/ingest/zeek.ntlm index c8f2e37c0..a7d0cf782 100644 --- a/salt/elasticsearch/files/ingest/zeek.ntlm +++ b/salt/elasticsearch/files/ingest/zeek.ntlm @@ -1,6 +1,7 @@ { "description" : "zeek.ntlm", "processors" : [ + { "set": { "field": "event.dataset", "value": "ntlm" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.hostname", "target_field": "host.name", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary b/salt/elasticsearch/files/ingest/zeek.opcua_binary index 37a9cdf1a..acb2e1b6b 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.msg_type", "target_field": "opcua.message_type", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session index 81f8e9392..00e3aa447 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_activate_session", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_activate_session" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true} }, { "rename": { "field": "message2.opcua_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_client_software_cert b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_client_software_cert index fe6f577eb..5f4f3d42d 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_client_software_cert +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_client_software_cert @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_activate_session_client_software_cert", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_activate_session_client_software_cert" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.client_software_cert_link_id", "target_field": "opcua.client_software_cert_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_diagnostic_info b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_diagnostic_info index 283d1c78e..5363fe73d 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_diagnostic_info +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_diagnostic_info @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_activate_session_diagnostic_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_activate_session_diagnostic_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.activate_session_diag_info_link_id", "target_field": "opcua.activate_session_diag_info_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_locale_id b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_locale_id index 08d8a672e..353047101 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_locale_id +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_activate_session_locale_id @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_activate_session_locale_id", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_activate_session_locale_id" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcua_locale_link_id", "target_field": "opcua.locale_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse index 8c4d919cd..6bb11c636 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_browse", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_browse" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcua_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_description b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_description index f1439f192..df35dff9b 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_description +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_description @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_browse_description", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_browse_description" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.browse_description_link_id", "target_field": "opcua.browse_description_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_diagnostic_info b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_diagnostic_info index 43560f4f0..71060bb6b 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_diagnostic_info +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_diagnostic_info @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_browse_diagnostic_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_browse_diagnostic_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.browse_diag_info_link_id", "target_field": "opcua.browse_session_diag_info_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_request_continuation_point b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_request_continuation_point index ce971109b..48e68edad 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_request_continuation_point +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_request_continuation_point @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_browse_request_continuation_point", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_browse_request_continuation_point" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.browse_next_link_id", "target_field": "opcua.browse_next_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_response_references b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_response_references index 960a0a939..ccfff9911 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_response_references +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_response_references @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_browse_response_references", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_browse_response_references" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.browse_reference_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_result b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_result index d9eabbd87..9d47076fd 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_result +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_browse_result @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_browse_result", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_browse_result" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.browse_response_link_id", "target_field": "opcua.response_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session index d7e0d3d87..d2e490f1f 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_create_session", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_create_session" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true} }, { "rename": { "field": "message2.opcua_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_discovery b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_discovery index cf9a56135..7a7753e45 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_discovery +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_discovery @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_create_session_discovery", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_create_session_discovery" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.discovery_profile_link_id", "target_field": "opcua.discovery_profile_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_endpoints b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_endpoints index 0511211d5..c1e599c1e 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_endpoints +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_endpoints @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_create_session_endpoints", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_create_session_endpoints" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.endpoint_link_id", "target_field": "opcua.endpoint_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_user_token b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_user_token index b86ec066d..169479a48 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_user_token +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_session_user_token @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_create_session_user_token", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_create_session_user_token" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.user_token_link_id", "target_field": "opcua.user_token_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_subscription b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_subscription index 832ac75b1..c282f9393 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_subscription +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_create_subscription @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_create_subscription", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_create_subscription" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcua_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_diag_info_detail b/salt/elasticsearch/files/ingest/zeek.opcua_binary_diag_info_detail index 170c35be0..9d6d63896 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_diag_info_detail +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_diag_info_detail @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_diag_info_detail", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_diag_info_detail" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.diag_info_link_id", "target_field": "opcua.diag_info_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints index 51f9349fc..90de3e82f 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_get_endpoints", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_get_endpoints" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcua_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_description b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_description index b467196de..de9c9b0bc 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_description +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_description @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_get_endpoints_description", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_get_endpoints_description" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.endpoint_description_link_id", "target_field": "opcua.endpoint_description_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_discovery b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_discovery index eeaf91dcb..4e9f964cf 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_discovery +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_discovery @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_get_endpoints_discovery", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_get_endpoints_discovery" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.discovery_profile_link_id", "target_field": "opcua.discovery_profile_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_locale_id b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_locale_id index 3716b3bb5..ea592e19e 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_locale_id +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_locale_id @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_get_endpoints_locale_id", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_get_endpoints_locale_id" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcua_locale_link_id", "target_field": "opcua.locale_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_profile_uri b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_profile_uri index 65309e588..39a9ef49a 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_profile_uri +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_profile_uri @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_get_endpoints_profile_uri", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_get_endpoints_profile_uri" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.profile_uri_link_id", "target_field": "opcua.profile_uri_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_user_token b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_user_token index 33a3687cc..30d160c9c 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_user_token +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_get_endpoints_user_token @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_get_endpoints_user_token", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_get_endpoints_user_token" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.user_token_link_id", "target_field": "opcua.user_token_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_opensecure_channel b/salt/elasticsearch/files/ingest/zeek.opcua_binary_opensecure_channel index 59c41206d..df91874f5 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_opensecure_channel +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_opensecure_channel @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_opensecure_channel", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_opensecure_channel" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcua_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read index 9eee12ff7..07a9134e7 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.opcua_link_id", "target_field": "opcua.link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims index a0955f534..661a4b49a 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_array_dims", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_array_dims" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.array_dim_link_id", "target_field": "opcua.array_dim_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims_link b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims_link index 94644246f..436aa65d5 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims_link +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_array_dims_link @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_array_dims_link", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_array_dims_link" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.variant_data_array_dim_link_id", "target_field": "opcua.variant_data_array_dim_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_diagnostic_info b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_diagnostic_info index 64376bd08..738c39e03 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_diagnostic_info +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_diagnostic_info @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_diagnostic_info", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_diagnostic_info" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.read_diag_info_link_id", "target_field": "opcua.read_diag_info_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object index 8ef46251b..b5f76e80b 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_extension_object", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_extension_object" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.ext_obj_link_id", "target_field": "opcua.ext_obj_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object_link b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object_link index 0aae27ca1..463fbdeb5 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object_link +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_extension_object_link @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_extension_object_link", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_extension_object_link" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.variant_data_ext_obj_link_id", "target_field": "opcua.variant_data_ext_obj_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_nodes_to_read b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_nodes_to_read index 39c0c25b1..71263a3be 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_nodes_to_read +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_nodes_to_read @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_nodes_to_read", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_nodes_to_read" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.nodes_to_read_link_id", "target_field": "opcua.nodes_to_read_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results index 49d14d404..d049c851b 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_results", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_results" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.results_link_id", "target_field": "opcua.results_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results_link b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results_link index 75245d212..a854b1f2f 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results_link +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_results_link @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_results_link", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_results_link" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.read_results_link_id", "target_field": "opcua.read_results_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_status_code b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_status_code index d3b6ece54..5c34f9b61 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_status_code +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_status_code @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_status_code", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_status_code" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.read_status_code_link_id", "target_field": "opcua.read_status_code_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data index d77404bc5..725efaf07 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_variant_data", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_variant_data" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.read_variant_data_link_id", "target_field": "opcua.read_variant_data_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data_link b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data_link index 8585789ff..caad459ed 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data_link +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_read_variant_data_link @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_read_variant_data_link", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_read_variant_data_link" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.read_results_variant_data_link_id", "target_field": "opcua.read_results_variant_data_link_id", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.opcua_binary_status_code_detail b/salt/elasticsearch/files/ingest/zeek.opcua_binary_status_code_detail index e1bff04a4..2a410da2d 100644 --- a/salt/elasticsearch/files/ingest/zeek.opcua_binary_status_code_detail +++ b/salt/elasticsearch/files/ingest/zeek.opcua_binary_status_code_detail @@ -1,6 +1,7 @@ { "description" : "zeek.opcua_binary_status_code_detail", "processors" : [ + { "set": { "field": "event.dataset", "value": "opcua_binary_status_code_detail" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.status_code", "target_field": "opcua.status_code", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.pe b/salt/elasticsearch/files/ingest/zeek.pe index 84f833742..e3bcd678f 100644 --- a/salt/elasticsearch/files/ingest/zeek.pe +++ b/salt/elasticsearch/files/ingest/zeek.pe @@ -1,6 +1,7 @@ { "description" : "zeek.pe", "processors" : [ + { "set": { "field": "event.dataset", "value": "pe" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.id", "target_field": "log.id.fuid", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.profinet b/salt/elasticsearch/files/ingest/zeek.profinet index e9d69c0dc..ca34a305c 100644 --- a/salt/elasticsearch/files/ingest/zeek.profinet +++ b/salt/elasticsearch/files/ingest/zeek.profinet @@ -1,6 +1,7 @@ { "description" : "zeek.profinet", "processors" : [ + { "set": { "field": "event.dataset", "value": "profinet" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true} }, { "rename": { "field": "message2.operation_type", "target_field": "profinet.operation_type", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.profinet_dce_rpc b/salt/elasticsearch/files/ingest/zeek.profinet_dce_rpc index e89fd7d95..c3e200630 100644 --- a/salt/elasticsearch/files/ingest/zeek.profinet_dce_rpc +++ b/salt/elasticsearch/files/ingest/zeek.profinet_dce_rpc @@ -1,6 +1,7 @@ { "description" : "zeek.profinet_dce_rpc", "processors" : [ + { "set": { "field": "event.dataset", "value": "profinet_dce_rpc" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true} }, { "rename": { "field": "message2.version", "target_field": "profinet.version", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.radius b/salt/elasticsearch/files/ingest/zeek.radius index 715f41478..3b25dd32b 100644 --- a/salt/elasticsearch/files/ingest/zeek.radius +++ b/salt/elasticsearch/files/ingest/zeek.radius @@ -1,6 +1,7 @@ { "description" : "zeek.radius", "processors" : [ + { "set": { "field": "event.dataset", "value": "radius" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.username", "target_field": "user.name", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.rdp b/salt/elasticsearch/files/ingest/zeek.rdp index ce792f32a..a5aea03f1 100644 --- a/salt/elasticsearch/files/ingest/zeek.rdp +++ b/salt/elasticsearch/files/ingest/zeek.rdp @@ -1,6 +1,7 @@ { "description" : "zeek.rdp", "processors" : [ + { "set": { "field": "event.dataset", "value": "rdp" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.cookie", "target_field": "rdp.cookie", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.rfb b/salt/elasticsearch/files/ingest/zeek.rfb index fc0653d45..c566827c8 100644 --- a/salt/elasticsearch/files/ingest/zeek.rfb +++ b/salt/elasticsearch/files/ingest/zeek.rfb @@ -1,6 +1,7 @@ { "description" : "zeek.rfb", "processors" : [ + { "set": { "field": "event.dataset", "value": "rfb" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.client_major_version", "target_field": "rfb.client_major_version", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.s7comm b/salt/elasticsearch/files/ingest/zeek.s7comm index e9f5e6318..0af7ee935 100644 --- a/salt/elasticsearch/files/ingest/zeek.s7comm +++ b/salt/elasticsearch/files/ingest/zeek.s7comm @@ -1,6 +1,7 @@ { "description" : "zeek.s7comm", "processors" : [ + { "set": { "field": "event.dataset", "value": "s7comm" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.rosctr_code", "target_field": "s7.ros.control.code", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.s7comm_plus b/salt/elasticsearch/files/ingest/zeek.s7comm_plus index cbb7d5723..e37603752 100644 --- a/salt/elasticsearch/files/ingest/zeek.s7comm_plus +++ b/salt/elasticsearch/files/ingest/zeek.s7comm_plus @@ -1,6 +1,7 @@ { "description" : "zeek.s7comm_plus", "processors" : [ + { "set": { "field": "event.dataset", "value": "s7comm_plus" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.version", "target_field": "s7.version", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.s7comm_read_szl b/salt/elasticsearch/files/ingest/zeek.s7comm_read_szl index c044c08a5..55571584a 100644 --- a/salt/elasticsearch/files/ingest/zeek.s7comm_read_szl +++ b/salt/elasticsearch/files/ingest/zeek.s7comm_read_szl @@ -1,6 +1,7 @@ { "description" : "zeek.s7comm_read_szl", "processors" : [ + { "set": { "field": "event.dataset", "value": "s7comm_read_szl" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.pdu_reference", "target_field": "s7.pdu_reference", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.s7comm_upload_download b/salt/elasticsearch/files/ingest/zeek.s7comm_upload_download index 51d8ca03f..83f773f0a 100644 --- a/salt/elasticsearch/files/ingest/zeek.s7comm_upload_download +++ b/salt/elasticsearch/files/ingest/zeek.s7comm_upload_download @@ -1,6 +1,7 @@ { "description" : "zeek.s7comm_upload_download", "processors" : [ + { "set": { "field": "event.dataset", "value": "s7comm_upload_download" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.rosctr", "target_field": "s7.ros.control.name", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.signatures b/salt/elasticsearch/files/ingest/zeek.signatures index 2a5f23ec7..00381660e 100644 --- a/salt/elasticsearch/files/ingest/zeek.signatures +++ b/salt/elasticsearch/files/ingest/zeek.signatures @@ -1,6 +1,7 @@ { "description" : "zeek.signatures", "processors" : [ + { "set": { "field": "event.dataset", "value": "signatures" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.note", "target_field": "note", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.sip b/salt/elasticsearch/files/ingest/zeek.sip index 5a8627878..a520f5bcb 100644 --- a/salt/elasticsearch/files/ingest/zeek.sip +++ b/salt/elasticsearch/files/ingest/zeek.sip @@ -1,6 +1,7 @@ { "description" : "zeek.sip", "processors" : [ + { "set": { "field": "event.dataset", "value": "sip" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.trans_depth", "target_field": "sip.transaction.depth", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.smb_files b/salt/elasticsearch/files/ingest/zeek.smb_files index da78953fe..192c6f766 100644 --- a/salt/elasticsearch/files/ingest/zeek.smb_files +++ b/salt/elasticsearch/files/ingest/zeek.smb_files @@ -1,6 +1,7 @@ { "description" : "zeek.smb_files", "processors" : [ + { "set": { "field": "event.dataset", "value": "smb_files" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.fuid", "target_field": "log.id.fuid", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.smb_mapping b/salt/elasticsearch/files/ingest/zeek.smb_mapping index 637ff414f..5c6d8bb89 100644 --- a/salt/elasticsearch/files/ingest/zeek.smb_mapping +++ b/salt/elasticsearch/files/ingest/zeek.smb_mapping @@ -1,6 +1,7 @@ { "description" : "zeek.smb_mapping", "processors" : [ + { "set": { "field": "event.dataset", "value": "smb_mapping" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.path", "target_field": "smb.path", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.smtp b/salt/elasticsearch/files/ingest/zeek.smtp index 9bfb1e3e1..e671cce4c 100644 --- a/salt/elasticsearch/files/ingest/zeek.smtp +++ b/salt/elasticsearch/files/ingest/zeek.smtp @@ -1,6 +1,7 @@ { "description" : "zeek.smtp", "processors" : [ + { "set": { "field": "event.dataset", "value": "smtp" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "remove": { "field": "path", "ignore_failure": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.snmp b/salt/elasticsearch/files/ingest/zeek.snmp index 664bc4112..6ee65f776 100644 --- a/salt/elasticsearch/files/ingest/zeek.snmp +++ b/salt/elasticsearch/files/ingest/zeek.snmp @@ -1,6 +1,7 @@ { "description" : "zeek.snmp", "processors" : [ + { "set": { "field": "event.dataset", "value": "snmp" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.duration", "target_field": "event.duration", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.socks b/salt/elasticsearch/files/ingest/zeek.socks index 6ccfa0d1a..24357c85b 100644 --- a/salt/elasticsearch/files/ingest/zeek.socks +++ b/salt/elasticsearch/files/ingest/zeek.socks @@ -1,6 +1,7 @@ { "description" : "zeek.socks", "processors" : [ + { "set": { "field": "event.dataset", "value": "socks" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.version", "target_field": "socks.version", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.software b/salt/elasticsearch/files/ingest/zeek.software index 16a2ae1f1..f5d3d1013 100644 --- a/salt/elasticsearch/files/ingest/zeek.software +++ b/salt/elasticsearch/files/ingest/zeek.software @@ -1,6 +1,7 @@ { "description" : "zeek.software", "processors" : [ + { "set": { "field": "event.dataset", "value": "software" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "dot_expander": { "field": "version.major", "path": "message2", "ignore_failure": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ssh b/salt/elasticsearch/files/ingest/zeek.ssh index 03a9abc15..cffcd854f 100644 --- a/salt/elasticsearch/files/ingest/zeek.ssh +++ b/salt/elasticsearch/files/ingest/zeek.ssh @@ -1,6 +1,7 @@ { "description" : "zeek.ssh", "processors" : [ + { "set": { "field": "event.dataset", "value": "ssh" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.version", "target_field": "ssh.version", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ssl b/salt/elasticsearch/files/ingest/zeek.ssl index e51fd4591..8ae7c8728 100644 --- a/salt/elasticsearch/files/ingest/zeek.ssl +++ b/salt/elasticsearch/files/ingest/zeek.ssl @@ -1,6 +1,7 @@ { "description" : "zeek.ssl", "processors" : [ + { "set": { "field": "event.dataset", "value": "ssl" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.version", "target_field": "ssl.version", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.stun b/salt/elasticsearch/files/ingest/zeek.stun index 48f648d74..322c1030a 100644 --- a/salt/elasticsearch/files/ingest/zeek.stun +++ b/salt/elasticsearch/files/ingest/zeek.stun @@ -1,6 +1,7 @@ { "description" : "zeek.stun", "processors" : [ + { "set": { "field": "event.dataset", "value": "stun" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true} }, { "rename": { "field": "message2.proto", "target_field": "network.protocol", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.stun_nat b/salt/elasticsearch/files/ingest/zeek.stun_nat index 45f9b3055..3ba4f7840 100644 --- a/salt/elasticsearch/files/ingest/zeek.stun_nat +++ b/salt/elasticsearch/files/ingest/zeek.stun_nat @@ -1,6 +1,7 @@ { "description" : "zeek.stun_nat", "processors" : [ + { "set": { "field": "event.dataset", "value": "stun_nat" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true} }, { "rename": { "field": "message2.proto", "target_field": "network.protocol", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.syslog b/salt/elasticsearch/files/ingest/zeek.syslog index 5bdbd9736..4d58430a3 100644 --- a/salt/elasticsearch/files/ingest/zeek.syslog +++ b/salt/elasticsearch/files/ingest/zeek.syslog @@ -1,6 +1,7 @@ { "description" : "zeek.syslog", "processors" : [ + { "set": { "field": "event.dataset", "value": "syslog" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.proto", "target_field": "network.protocol", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.tds b/salt/elasticsearch/files/ingest/zeek.tds index 43c2cad18..9ad7b60bb 100644 --- a/salt/elasticsearch/files/ingest/zeek.tds +++ b/salt/elasticsearch/files/ingest/zeek.tds @@ -1,6 +1,7 @@ { "description" : "zeek.tds", "processors" : [ + { "set": { "field": "event.dataset", "value": "tds" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.command", "target_field": "tds.command", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.tds_rpc b/salt/elasticsearch/files/ingest/zeek.tds_rpc index 75a73c6ba..2915653c3 100644 --- a/salt/elasticsearch/files/ingest/zeek.tds_rpc +++ b/salt/elasticsearch/files/ingest/zeek.tds_rpc @@ -1,6 +1,7 @@ { "description" : "zeek.tds_rpc", "processors" : [ + { "set": { "field": "event.dataset", "value": "tds_rpc" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.procedure_name", "target_field": "tds.procedure_name", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.tds_sql_batch b/salt/elasticsearch/files/ingest/zeek.tds_sql_batch index 560cd1ef3..70f3ef755 100644 --- a/salt/elasticsearch/files/ingest/zeek.tds_sql_batch +++ b/salt/elasticsearch/files/ingest/zeek.tds_sql_batch @@ -1,6 +1,7 @@ { "description" : "zeek.tds_sql_batch", "processors" : [ + { "set": { "field": "event.dataset", "value": "tds_sql_batch" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.header_type", "target_field": "tds.header_type", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.tunnel b/salt/elasticsearch/files/ingest/zeek.tunnel index 66250091e..76b1ca745 100644 --- a/salt/elasticsearch/files/ingest/zeek.tunnel +++ b/salt/elasticsearch/files/ingest/zeek.tunnel @@ -1,6 +1,7 @@ { "description" : "zeek.tunnel", "processors" : [ + { "set": { "field": "event.dataset", "value": "tunnel" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "pipeline": { "name": "zeek.tunnels" } } ] diff --git a/salt/elasticsearch/files/ingest/zeek.tunnels b/salt/elasticsearch/files/ingest/zeek.tunnels index bb4c18268..84fddcda1 100644 --- a/salt/elasticsearch/files/ingest/zeek.tunnels +++ b/salt/elasticsearch/files/ingest/zeek.tunnels @@ -1,6 +1,7 @@ { "description" : "zeek.tunnels", "processors" : [ + { "set": { "field": "event.dataset", "value": "tunnels" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.tunnel_type", "target_field": "tunnel.type", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.weird b/salt/elasticsearch/files/ingest/zeek.weird index 2665bbb33..deb5e8058 100644 --- a/salt/elasticsearch/files/ingest/zeek.weird +++ b/salt/elasticsearch/files/ingest/zeek.weird @@ -1,6 +1,7 @@ { "description" : "zeek.weird", "processors" : [ + { "set": { "field": "event.dataset", "value": "weird" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.name", "target_field": "weird.name", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.wireguard b/salt/elasticsearch/files/ingest/zeek.wireguard index 1df929666..26ee0e0a3 100644 --- a/salt/elasticsearch/files/ingest/zeek.wireguard +++ b/salt/elasticsearch/files/ingest/zeek.wireguard @@ -1,6 +1,7 @@ { "description" : "zeek.wireguard", "processors" : [ + { "set": { "field": "event.dataset", "value": "wireguard" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.established", "target_field": "wireguard.established", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.x509 b/salt/elasticsearch/files/ingest/zeek.x509 index 49a79dbd0..640ea81e3 100644 --- a/salt/elasticsearch/files/ingest/zeek.x509 +++ b/salt/elasticsearch/files/ingest/zeek.x509 @@ -1,6 +1,7 @@ { "description" : "zeek.x509", "processors" : [ + { "set": { "field": "event.dataset", "value": "x509" } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.id", "target_field": "log.id.fuid", "ignore_missing": true } }, diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 9014f7a3b..82f183b3a 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -474,6 +474,7 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} + - {{ portgroups.elastic_agent_control }} sensors: portgroups: - {{ portgroups.beats_5044 }} @@ -491,6 +492,10 @@ role: elasticsearch_rest: portgroups: - {{ portgroups.elasticsearch_rest }} + elastic_agent_endpoint: + portgroups: + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} analyst: portgroups: - {{ portgroups.nginx }} diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 5a4936843..1758f17ae 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -94,10 +94,10 @@ wait_for_influxdb: - ssl: True - verify_ssl: False - status: 200 - - timeout: 30 + - timeout: 10 - retry: - attempts: 5 - interval: 60 + attempts: 20 + interval: 5 - require: - docker_container: so-influxdb diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index dadb48ffe..2a4e85b9d 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.5.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.6.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index 409bc9209..3ac67ed27 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.5.3","id": "8.5.3","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.6.1","id": "8.6.1","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} diff --git a/salt/logstash/pipelines/config/so/0009_input_beats.conf b/salt/logstash/pipelines/config/so/0009_input_beats.conf deleted file mode 100644 index 8643a64b4..000000000 --- a/salt/logstash/pipelines/config/so/0009_input_beats.conf +++ /dev/null @@ -1,11 +0,0 @@ -input { - beats { - port => "5044" - tags => [ "beat-ext" ] - } -} -filter { - mutate { - rename => {"@metadata" => "metadata"} - } -} diff --git a/salt/logstash/pipelines/config/so/0010_input_hhbeats.conf b/salt/logstash/pipelines/config/so/0010_input_hhbeats.conf deleted file mode 100644 index 050d01d73..000000000 --- a/salt/logstash/pipelines/config/so/0010_input_hhbeats.conf +++ /dev/null @@ -1,40 +0,0 @@ -input { - beats { - port => "5644" - ssl => true - ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"] - ssl_certificate => "/usr/share/logstash/filebeat.crt" - ssl_key => "/usr/share/logstash/filebeat.key" - #tags => [ "beat" ] - } -} -filter { - if [type] == "ids" or [type] =~ "bro" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_field => { "sensor_name" => "%{[beat][name]}" } - add_field => { "syslog-host_from" => "%{[beat][name]}" } - remove_field => [ "beat", "prospector", "input", "offset" ] - } - } - if [type] =~ "ossec" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_field => { "syslog-host_from" => "%{[beat][name]}" } - remove_field => [ "beat", "prospector", "input", "offset" ] - } - } - if [type] == "osquery" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_tag => ["osquery"] - } - json { - source => "message" - target => "osquery" - } - } -} diff --git a/salt/logstash/pipelines/config/so/0800_input_eval.conf b/salt/logstash/pipelines/config/so/0800_input_eval.conf deleted file mode 100644 index 35a977d04..000000000 --- a/salt/logstash/pipelines/config/so/0800_input_eval.conf +++ /dev/null @@ -1,204 +0,0 @@ -# Updated by: Mike Reeves -# Last Update: 11/1/2018 - -input { - file { - path => "/suricata/eve.json" - type => "ids" - add_field => { "engine" => "suricata" } - } - file { - path => "/nsm/zeek/logs/current/conn*.log" - type => "zeek.conn" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dce_rpc*.log" - type => "zeek.dce_rpc" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dhcp*.log" - type => "zeek.dhcp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dnp3*.log" - type => "zeek.dnp3" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dns*.log" - type => "zeek.dns" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dpd*.log" - type => "zeek.dpd" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/files*.log" - type => "zeek.files" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ftp*.log" - type => "zeek.ftp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/http*.log" - type => "zeek.http" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/intel*.log" - type => "zeek.intel" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/irc*.log" - type => "zeek.irc" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/kerberos*.log" - type => "zeek.kerberos" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/modbus*.log" - type => "zeek.modbus" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/mysql*.log" - type => "zeek.mysql" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/notice*.log" - type => "zeek.notice" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ntlm*.log" - type => "zeek.ntlm" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/pe*.log" - type => "zeek.pe" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/radius*.log" - type => "zeek.radius" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/rdp*.log" - type => "zeek.rdp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/rfb*.log" - type => "zeek.rfb" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/signatures*.log" - type => "zeek.signatures" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/sip*.log" - type => "zeek.sip" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/smb_files*.log" - type => "zeek.smb_files" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/smb_mapping*.log" - type => "zeek.smb_mapping" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/smtp*.log" - type => "zeek.smtp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/snmp*.log" - type => "zeek.snmp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/socks*.log" - type => "zeek.socks" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/software*.log" - type => "zeek.software" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ssh*.log" - type => "zeek.ssh" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ssl*.log" - type => "zeek.ssl" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/syslog*.log" - type => "zeek.syslog" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/tunnel*.log" - type => "zeek.tunnels" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/weird*.log" - type => "zeek.weird" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/x509*.log" - type => "zeek.x509" - tags => ["zeek"] - } - file { - path => "/wazuh/alerts/alerts.json" - type => "ossec" - } -# file { -# path => "/wazuh/archives/archives.json" -# type => "ossec_archive" -# } - file { - path => "/osquery/logs/result.log" - type => "osquery" - tags => ["osquery"] - } - file { - path => "/strelka/strelka.log" - type => "strelka" - } -} -filter { - if "import" in [tags] { - mutate { - #add_tag => [ "conf_file_0007"] - } - } -} diff --git a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja deleted file mode 100644 index 7a0848b39..000000000 --- a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja +++ /dev/null @@ -1,23 +0,0 @@ -{%- if grains.role == 'so-heavynode' %} -{%- set HOST = GLOBALS.hostname %} -{%- else %} -{%- set HOST = GLOBALS.manager %} -{% endif -%} -{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} -{%- set access_key = salt['pillar.get']('minio:access_key', '') %} -{%- set access_secret = salt['pillar.get']('minio:access_secret', '') %} -{%- set INTERVAL = salt['pillar.get']('s3_settings:interval', 5) %} -input { - s3 { - access_key_id => "{{ access_key }}" - secret_access_key => "{{ access_secret }}" - endpoint => "https://{{ HOST }}:9595" - bucket => "logstash" - delete => true - interval => {{ INTERVAL }} - codec => json - additional_settings => { - "force_path_style" => true - } - } -} diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja deleted file mode 100644 index 7b8c03f45..000000000 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "zeek" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-zeek" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja deleted file mode 100644 index a57830229..000000000 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if "import" in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-import" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja deleted file mode 100644 index 4c49c61ea..000000000 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "syslog" { - elasticsearch { - pipeline => "%{module}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-syslog" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja deleted file mode 100644 index 672a83876..000000000 --- a/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja +++ /dev/null @@ -1,14 +0,0 @@ -output { - if "filebeat" in [metadata][pipeline] { - elasticsearch { - id => "filebeat_modules_metadata_pipeline" - pipeline => "%{[metadata][pipeline]}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-%{[event][module]}-%{+YYYY.MM.dd}" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja deleted file mode 100644 index 8dbea872e..000000000 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "osquery" and "live_query" not in [dataset] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-osquery" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja deleted file mode 100644 index 7942aa50c..000000000 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ /dev/null @@ -1,12 +0,0 @@ -output { - if [dataset] =~ "firewall" { - elasticsearch { - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-firewall" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja deleted file mode 100644 index 13df33e16..000000000 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "suricata" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-ids" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja deleted file mode 100644 index b4aafecad..000000000 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ /dev/null @@ -1,26 +0,0 @@ -output { - if "beat-ext" in [tags] and "import" not in [tags] and "filebeat" not in [metadata][pipeline] { - if [metadata][_id] { - elasticsearch { - pipeline => "beats.common" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-beats" - ssl => true - ssl_certificate_verification => false - document_id => "%{[metadata][_id]}" - } - } else { - elasticsearch { - pipeline => "beats.common" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-beats" - ssl => true - ssl_certificate_verification => false - } - } - } -} diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja deleted file mode 100644 index ca3eeb6c1..000000000 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "ossec" { - elasticsearch { - pipeline => "%{module}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-ossec" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja deleted file mode 100644 index 281cdda5b..000000000 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ /dev/null @@ -1,14 +0,0 @@ -output { - if [module] =~ "strelka" { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-strelka" - ssl => true - ssl_certificate_verification => false - } - } -} - diff --git a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja deleted file mode 100644 index 8127de23a..000000000 --- a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja +++ /dev/null @@ -1,14 +0,0 @@ -output { - if [module] =~ "logscan" { - elasticsearch { - id => "logscan_pipeline" - pipeline => "logscan.alert" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-logscan" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja b/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja deleted file mode 100644 index 7f9d795e6..000000000 --- a/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "rita" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-rita" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja index 1c770c7b0..0a148155c 100644 --- a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja +++ b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja @@ -1,5 +1,5 @@ output { - if "elastic-agent" in [tags] and "import" not in [tags] { + if "elastic-agent" in [tags] { if [metadata][pipeline] { elasticsearch { hosts => "{{ GLOBALS.manager }}" diff --git a/salt/mysql/etc/my.cnf b/salt/mysql/etc/my.cnf index e37f690dc..243cc4c51 100644 --- a/salt/mysql/etc/my.cnf +++ b/salt/mysql/etc/my.cnf @@ -29,3 +29,6 @@ symbolic-links=0 log-error=/var/log/mysql/mysqld.log pid-file=/var/run/mysqld/mysqld.pid + +# Switch back to the native password module so that playbook can connect +default_authentication_plugin=mysql_native_password \ No newline at end of file diff --git a/salt/repo/client/ubuntu.sls b/salt/repo/client/ubuntu.sls new file mode 100644 index 000000000..a5d55fa5e --- /dev/null +++ b/salt/repo/client/ubuntu.sls @@ -0,0 +1,20 @@ +# this removes the repo file left by bootstrap-salt.sh without -r +remove_salt.list: + file.absent: + - name: /etc/apt/sources.list.d/salt.list + +saltstack.list: + file.managed: + - name: /etc/apt/sources.list.d/saltstack.list + - contents: + - deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/{{grains.osrelease}}/amd64/salt3004.2/ {{grains.oscodename}} main + +apt_update: + cmd.run: + - name: apt-get update + - onchanges: + - file: saltstack.list + - timeout: 30 + - retry: + attempts: 5 + interval: 30 \ No newline at end of file diff --git a/salt/sensoroni/files/analyzers/README.md b/salt/sensoroni/files/analyzers/README.md index e87a95638..8b1f44f29 100644 --- a/salt/sensoroni/files/analyzers/README.md +++ b/salt/sensoroni/files/analyzers/README.md @@ -3,6 +3,7 @@ Security Onion provides a means for performing data analysis on varying inputs. This data can be any data of interest sourced from event logs. Examples include hostnames, IP addresses, file hashes, URLs, etc. The analysis is conducted by one or more analyzers that understand that type of input. Analyzers come with the default installation of Security Onion. However, it is also possible to add additional analyzers to extend the analysis across additional areas or data types. ## Supported Observable Types + The built-in analyzers support the following observable types: | Name | Domain | Hash | IP | Mail | Other | URI | URL | User Agent | @@ -20,6 +21,7 @@ The built-in analyzers support the following observable types: | WhoisLookup |✓ |✗|✗|✗|✗|✓|✗|✗| ## Authentication + Many analyzers require authentication, via an API key or similar. The table below illustrates which analyzers require authentication. | Name | Authn Req'd| diff --git a/salt/setup/highstate_cron.sls b/salt/setup/highstate_cron.sls new file mode 100644 index 000000000..dcbab4d8d --- /dev/null +++ b/salt/setup/highstate_cron.sls @@ -0,0 +1,6 @@ +post_setup_cron: + cron.present: + - name: 'salt-call state.highstate' + - user: root + - minute: '*/1' + - identifier: post_setup_cron diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 857f245d1..cec11273b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1020,6 +1020,9 @@ soc: cacheMs: 300000 verifyCert: false casesEnabled: true + extractCommonObservables: + - source.ip + - destination.ip timeoutMs: 300000 timeShiftMs: 120000 defaultDurationMs: 1800000 diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 40cb4487d..3c1000dee 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -110,12 +110,13 @@ so-soc: - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw - /opt/so/conf/soc/salt:/opt/sensoroni/salt:rw - /opt/so/saltstack:/opt/so/saltstack:rw - {%- if salt['pillar.get']('nodestab', {}) %} - extra_hosts: + - {{GLOBALS.influxdb_host}}:{{pillar.node_data[GLOBALS.influxdb_host].ip}} + {%- if salt['pillar.get']('nodestab', {}) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - {{ SN.split('_')|first }}:{{ SNDATA.ip }} {%- endfor %} - {%- endif %} + {%- endif %} - port_bindings: {% for BINDING in DOCKER.containers['so-soc'].port_bindings %} - {{ BINDING }} diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 7d96ca46b..e6b43cf0b 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -69,6 +69,9 @@ soc: description: Set to true if the SOC case management module, natively integrated with Elasticsearch, should be enabled. global: True advanced: True + extractCommonObservables: + description: List of indexed fields to automatically extract into a case observable, when attaching related events to a case. + global: True timeShiftMs: description: Duration (in milliseconds) to further expand the PCAP time range when querying PCAP data related to an event. This duration is added to the normal duration value (see defaultDurationMs). global: True diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index c9cbe70e5..0c3957a89 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -239,19 +239,6 @@ password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" {%- endif %} -{# if grains.role in ['so-eval','so-standalone','so-manager', 'so-managersearch', 'so-heavynode', 'so-receiver'] -%} -[[inputs.redis]] - servers = ["tcp://localhost:6379"] -{%- endif #} - -{%- if grains.role in ['so-searchnode', 'so-sensor', 'so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode'] %} -[[inputs.beat]] - url = "http://127.0.0.1:5066" - include = ["filebeat", "libbeat"] - username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_beats_user:user') }}" - password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_beats_user:pass') }}" -{%- endif %} - # # Read metrics from one or more commands that can output to stdout [[inputs.exec]] commands = [ diff --git a/salt/top.sls b/salt/top.sls index 09f1435ad..9524d5964 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -344,6 +344,7 @@ base: - zeek - schedule - docker_clean + - elastic-fleet '*_receiver and G@saltversion:{{saltversion}}': - match: compound diff --git a/salt/vars/manager.map.jinja b/salt/vars/manager.map.jinja new file mode 100644 index 000000000..c6b348341 --- /dev/null +++ b/salt/vars/manager.map.jinja @@ -0,0 +1,15 @@ +{% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %} +{% from 'vars/logstash.map.jinja' import LOGSTASH_GLOBALS %} + +{% set ROLE_GLOBALS = {} %} + +{% set MANAGER_GLOBALS = + [ + ELASTICSEARCH_GLOBALS, + LOGSTASH_GLOBALS + ] +%} + +{% for sg in MANAGER_GLOBALS %} +{% do salt['defaults.merge'](ROLE_GLOBALS, sg, merge_lists=False, in_place=True) %} +{% endfor %} diff --git a/setup/so-functions b/setup/so-functions index a046b49ca..a76126519 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -200,7 +200,7 @@ check_service_status() { } check_web_pass() { - info Making sure web credential passwords match + info "Making sure web credential passwords match" check_pass_match "$WEBPASSWD1" "$WEBPASSWD2" "WPMATCH" } @@ -383,16 +383,11 @@ collect_mngr_hostname() { fi } -collect_net_method() { +collect_net_method() { whiptail_net_method - - if [[ "$network_traffic" == *"_MANAGER" ]]; then - whiptail_manager_updates_warning - MANAGERUPDATES=1 - fi - if [[ "$network_traffic" == "PROXY"* ]]; then collect_proxy no_ask + needs_proxy=true fi } @@ -909,10 +904,9 @@ detect_os() { elif [ -f /etc/os-release ]; then OS=ubuntu - if grep -q "UBUNTU_CODENAME=bionic" /etc/os-release; then - OSVER=bionic - elif grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then + if grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then OSVER=focal + is_ubuntu=true else info "We do not support your current version of Ubuntu." exit 1 @@ -1307,15 +1301,7 @@ idh_pillar() { logstash_pillar() { # Create the logstash advanced pillar touch $adv_logstash_pillar_file - title "Create the logstash pillar" - printf '%s\n'\ - "logstash_settings:"\ - " ls_host: '$HOSTNAME'"\ - " ls_pipeline_batch_size: 125"\ - " ls_input_threads: 1"\ - " lsheap: $NODE_LS_HEAP_SIZE"\ - " ls_pipeline_workers: $num_cpu_cores"\ - "" > "$logstash_pillar_file" + touch $logstash_pillar_file } # Set Logstash heap size based on total memory @@ -1339,10 +1325,6 @@ ls_heapsize() { esac export LS_HEAP_SIZE - if [[ "$install_type" =~ ^(EVAL|MANAGERSEARCH|STANDALONE)$ ]]; then - NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE - export NODE_LS_HEAP_SIZE - fi } idstools_pillar() { @@ -1887,6 +1869,9 @@ drop_install_options() { NODETYPE=${install_type^^} echo "NODETYPE=$NODETYPE" >> /opt/so/install.txt echo "CORECOUNT=$lb_procs" >> /opt/so/install.txt + echo "LSHOSTNAME=$HOSTNAME" >> /opt/so/install.txt + echo "LSHEAP=$LS_HEAP_SIZE" >> /opt/so/install.txt + echo "CPUCORES=$num_cpu_cores" >> /opt/so/install.txt } remove_package() { @@ -1992,6 +1977,43 @@ repo_sync_local() { } saltify() { + if [[ $is_ubuntu ]]; then + + DEBIAN_FRONTEND=noninteractive retry 150 20 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || exit 1 + update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10 + local pkg_arr=( + 'apache2-utils' + 'ca-certificates' + 'curl' + 'software-properties-common' + 'apt-transport-https' + 'openssl' + 'netcat' + 'jq' + ) + retry 150 20 "apt-get -y install ${pkg_arr[*]}" || exit 1 + + logCmd "mkdir -vp /opt/so/gpg" + logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt/SALTSTACK-GPG-KEY.pub" + logCmd "wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg" + logCmd "apt-key add /opt/so/gpg/SALTSTACK-GPG-KEY.pub" + logCmd "apt-key add /opt/so/gpg/docker.pub" + + # Add SO Saltstack Repo + echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt3004.2/ focal main" > /etc/apt/sources.list.d/saltstack.list + + # Add Docker Repo + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + # Ain't nothing but a GPG + + retry 150 20 "apt-get update" "" "Err:" || exit 1 + retry 150 20 "apt-get -y install salt-minion=3004.2+ds-1 salt-common=3004.2+ds-1" || exit 1 + retry 150 20 "apt-mark hold salt-minion salt-common" || exit 1 + retry 150 20 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb python3-lxml" || exit 1 + + fi + if [[ $is_centos ]]; then RUSALTY=$(rpm -qa | grep salt-minion | wc -l) if [[ "$RUSALTY" -gt 0 ]]; then @@ -2305,6 +2327,22 @@ so_add_user() { fi } +ubuntu_check() { + if [[ $OS == "ubuntu" ]]; then + if [[ $waitforstate ]]; then + whiptail_ubuntu_notsupported + exit 1 + else + if [[ $UBUNTUINSTALL == "needtoupgrade" ]]; then + whiptail_ubuntu_warning + else + whiptail_ubuntu_notsupported + exit 1 + fi + fi + fi +} + update_sudoers_for_testing() { if [ -n "$TESTING" ]; then info "Ensuring $INSTALLUSERNAME has password-less sudo access for automated testing purposes." @@ -2383,12 +2421,13 @@ wait_for_salt_minion() { verify_setup() { info "Verifying setup" - output=$(./so-verify "$setup_type" 2>&1) + set -o pipefail + ./so-verify "$setup_type" 2>&1 | tee -a $setup_log result=$? - echo "$output" >> "$setup_log" + set +o pipefail if [[ $result -eq 0 ]]; then whiptail_setup_complete else whiptail_setup_failed fi -} \ No newline at end of file +} diff --git a/setup/so-setup b/setup/so-setup index 1583e4b66..8e3af00b9 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -142,6 +142,7 @@ if [[ -f /root/accept_changes ]]; then mv "$setup_log" "$setup_log.bak" [ -f "$error_log" ] && mv "$error_log" "$error_log.bak" reinstall_init + reset_proxy fi title "Parsing Username for Install" @@ -314,10 +315,12 @@ if ! [[ -f $install_opt_file ]]; then # If you are a manager ask ALL the manager things here. I know there is code re-use but this makes it easier to add new roles. if [[ $is_eval ]]; then waitforstate=true + ubuntu_check monints=true check_elastic_license check_requirements "manager" networking_needful + collect_net_method collect_dockernet if [[ $is_iso ]]; then whiptail_airgap @@ -336,10 +339,12 @@ if ! [[ -f $install_opt_file ]]; then whiptail_end_settings elif [[ $is_standalone ]]; then waitforstate=true + ubuntu_check monints=true check_elastic_license check_requirements "manager" networking_needful + collect_net_method collect_dockernet if [[ $is_iso ]]; then whiptail_airgap @@ -359,13 +364,16 @@ if ! [[ -f $install_opt_file ]]; then elif [[ $is_manager ]]; then check_elastic_license waitforstate=true + ubuntu_check check_requirements "manager" networking_needful + collect_net_method collect_dockernet if [[ $is_iso ]]; then whiptail_airgap fi detect_cloud + set_minion_info set_default_log_size >> $setup_log 2>&1 info "Verifying all network devices are managed by Network Manager that should be" check_network_manager_conf @@ -378,13 +386,16 @@ if ! [[ -f $install_opt_file ]]; then elif [[ $is_managersearch ]]; then check_elastic_license waitforstate=true + ubuntu_check check_requirements "manager" networking_needful + collect_net_method collect_dockernet if [[ $is_iso ]]; then whiptail_airgap fi detect_cloud + set_minion_info set_default_log_size >> $setup_log 2>&1 info "Verifying all network devices are managed by Network Manager that should be" check_network_manager_conf @@ -395,6 +406,7 @@ if ! [[ -f $install_opt_file ]]; then collect_so_allow whiptail_end_settings elif [[ $is_sensor ]]; then + ubuntu_check monints=true check_requirements "sensor" calculate_useable_cores @@ -422,6 +434,7 @@ if ! [[ -f $install_opt_file ]]; then whiptail_end_settings elif [[ $is_searchnode ]]; then + ubuntu_check check_requirements "elasticsearch" networking_needful check_network_manager_conf @@ -434,6 +447,7 @@ if ! [[ -f $install_opt_file ]]; then whiptail_end_settings elif [[ $is_heavynode ]]; then + ubuntu_check monints=true check_requirements "heavynode" calculate_useable_cores @@ -441,17 +455,21 @@ if ! [[ -f $install_opt_file ]]; then collect_mngr_hostname add_mngr_ip_to_hosts check_manager_connection + set_minion_info whiptail_end_settings elif [[ $is_idh ]]; then + ubuntu_check check_requirements "idh" networking_needful collect_mngr_hostname add_mngr_ip_to_hosts check_manager_connection + set_minion_info whiptail_end_settings elif [[ $is_import ]]; then + ubuntu_check waitforstate=true monints=true check_elastic_license @@ -473,11 +491,13 @@ if ! [[ -f $install_opt_file ]]; then whiptail_end_settings elif [[ $is_receiver ]]; then + ubuntu_check check_requirements "receiver" networking_needful collect_mngr_hostname add_mngr_ip_to_hosts check_manager_connection + set_minion_info whiptail_end_settings fi @@ -487,6 +507,9 @@ if ! [[ -f $install_opt_file ]]; then percentage=0 es_heapsize ls_heapsize + if [[ $needs_proxy ]]; then + set_proxy + fi set_redirect # Generate Interface Vars generate_interface_vars @@ -535,6 +558,9 @@ if ! [[ -f $install_opt_file ]]; then export PATCHSCHEDULENAME=$PATCHSCHEDULENAME export INTERFACE="bond0" export CORECOUNT=$lb_procs + export LSHOSTNAME=$HOSTNAME + export LSHEAP=$LS_HEAP_SIZE + export CPUCORES=$num_cpu_cores logCmd "so-minion -o=setup" title "Creating Global SLS" @@ -589,9 +615,9 @@ if ! [[ -f $install_opt_file ]]; then add_web_user info "Restarting SOC to pick up initial user" logCmd "so-soc-restart" + title "Setting up Elastic Fleet" + logCmd "so-elastic-fleet-setup" if [[ ! $is_import ]]; then - title "Setting up Elastic Fleet" - logCmd "so-elastic-fleet-setup" title "Setting up Playbook" logCmd "so-playbook-reset" fi @@ -620,6 +646,7 @@ if ! [[ -f $install_opt_file ]]; then configure_minion "$minion_type" drop_install_options checkin_at_boot + logCmd "salt-call state.apply setup.highstate_cron --local --file-root=../salt/" verify_setup fi diff --git a/setup/so-verify b/setup/so-verify index 685e50d81..44c0465ff 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -17,7 +17,7 @@ using_iso() { if [ "$setup_type" == "iso" ]; then return 0 fi - return 0 + return 1 } # Check entire setup log for errors or unexpected salt states @@ -36,6 +36,7 @@ log_has_errors() { grep -vE "The Salt Master has cached the public key for this node" | \ grep -vE "Minion failed to authenticate with the master" | \ grep -vE "Failed to connect to ::1" | \ + grep -vE "Failed to set locale" | \ grep -vE "perl-Error-" | \ grep -vE "Failed:\s*?[0-9]+" | \ grep -vE "Status .* was not found" | \ @@ -54,7 +55,8 @@ log_has_errors() { # For ISO installs, we know nothing else can be running on this server, so there should be # nothing in any mail spool dir. cron_error_in_mail_spool() { - if find /var/spool/mail/ -type f -size +0 &> /dev/null; then + count=$(find /var/spool/mail/ -type f -size +0 | wc -l) + if [[ $count -ne 0 ]]; then return 0 fi return 1 diff --git a/setup/so-whiptail b/setup/so-whiptail index 5cc1d0c6f..b5967280e 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -816,34 +816,15 @@ whiptail_net_method() { ) local proxy_desc="proxy the traffic for git, docker client, wget, curl, ${pkg_mngr}, and various other SO components through a separate server in your environment." - if [[ $is_minion ]]; then - read -r -d '' options_msg <<- EOM + read -r -d '' options_msg <<- EOM ${options_msg} - - "Direct + Manager" - all traffic passes to the Internet normally, but ${pkg_mngr} updates will instead be pulled from ${mngr_article} manager. - "Proxy" - ${proxy_desc} + EOM + options+=( + " Proxy " "" + ) - "Proxy + Manager" - proxy all traffic from the "Proxy" option except ${pkg_mngr} updates, which will instead pull from the manager. - EOM - - options+=( - " Direct + Manager " "" - " Proxy " "" - " Proxy + Manager " "" - ) - local height=25 - else - read -r -d '' options_msg <<- EOM - ${options_msg} - - "Proxy" - ${proxy_desc} - EOM - options+=( - " Proxy " "" - ) - local height=17 - fi + local height=17 local msg read -r -d '' msg <<- EOM @@ -1327,6 +1308,28 @@ whiptail_storage_requirements() { whiptail_check_exitstatus $exitstatus } +whiptail_ubuntu_notsupported() { + [ -n "$TESTING" ] && return + + read -r -d '' message <<- EOM + Ubuntu is not supported for this node type. + + Please use a supported OS or install via ISO. + EOM + whiptail --title "$whiptail_title" --msgbox "$message" 14 75 +} + +whiptail_ubuntu_warning() { + [ -n "$TESTING" ] && return + + read -r -d '' message <<- EOM + Ubuntu support for this node type is limited. + + Please consider using a fully supported OS or install via ISO. + EOM + whiptail --title "$whiptail_title" --msgbox "$message" 14 75 + +} whiptail_uppercase_warning() { [ -n "$TESTING" ] && return