diff --git a/.github/.gitleaks.toml b/.github/.gitleaks.toml index 024b8ce51..21a047959 100644 --- a/.github/.gitleaks.toml +++ b/.github/.gitleaks.toml @@ -536,11 +536,10 @@ secretGroup = 4 [allowlist] description = "global allow lists" -regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*'''] +regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''', '''ssl_.*password'''] paths = [ '''gitleaks.toml''', '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''', '''(go.mod|go.sum)$''', - '''salt/nginx/files/enterprise-attack.json''' ] diff --git a/.github/DISCUSSION_TEMPLATE/2-4.yml b/.github/DISCUSSION_TEMPLATE/2-4.yml new file mode 100644 index 000000000..8e2592071 --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/2-4.yml @@ -0,0 +1,190 @@ +body: + - type: markdown + attributes: + value: | + ⚠️ This category is solely for conversations related to Security Onion 2.4 ⚠️ + + If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support + - type: dropdown + attributes: + label: Version + description: Which version of Security Onion 2.4.x are you asking about? + options: + - + - 2.4 Pre-release (Beta, Release Candidate) + - 2.4.10 + - 2.4.20 + - 2.4.30 + - 2.4.40 + - 2.4.50 + - 2.4.60 + - 2.4.70 + - 2.4.80 + - 2.4.90 + - 2.4.100 + - Other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Installation Method + description: How did you install Security Onion? + options: + - + - Security Onion ISO image + - Network installation on Red Hat derivative like Oracle, Rocky, Alma, etc. + - Network installation on Ubuntu + - Network installation on Debian + - Other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Description + description: > + Is this discussion about installation, configuration, upgrading, or other? + options: + - + - installation + - configuration + - upgrading + - other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Installation Type + description: > + When you installed, did you choose Import, Eval, Standalone, Distributed, or something else? + options: + - + - Import + - Eval + - Standalone + - Distributed + - other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Location + description: > + Is this deployment in the cloud, on-prem with Internet access, or airgap? + options: + - + - cloud + - on-prem with Internet access + - airgap + - other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Hardware Specs + description: > + Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://docs.securityonion.net/en/2.4/hardware.html? + options: + - + - Meets minimum requirements + - Exceeds minimum requirements + - Does not meet minimum requirements + - other (please provide detail below) + validations: + required: true + - type: input + attributes: + label: CPU + description: How many CPU cores do you have? + validations: + required: true + - type: input + attributes: + label: RAM + description: How much RAM do you have? + validations: + required: true + - type: input + attributes: + label: Storage for / + description: How much storage do you have for the / partition? + validations: + required: true + - type: input + attributes: + label: Storage for /nsm + description: How much storage do you have for the /nsm partition? + validations: + required: true + - type: dropdown + attributes: + label: Network Traffic Collection + description: > + Are you collecting network traffic from a tap or span port? + options: + - + - tap + - span port + - other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Network Traffic Speeds + description: > + How much network traffic are you monitoring? + options: + - + - Less than 1Gbps + - 1Gbps to 10Gbps + - more than 10Gbps + validations: + required: true + - type: dropdown + attributes: + label: Status + description: > + Does SOC Grid show all services on all nodes as running OK? + options: + - + - Yes, all services on all nodes are running OK + - No, one or more services are failed (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Salt Status + description: > + Do you get any failures when you run "sudo salt-call state.highstate"? + options: + - + - Yes, there are salt failures (please provide detail below) + - No, there are no failures + validations: + required: true + - type: dropdown + attributes: + label: Logs + description: > + Are there any additional clues in /opt/so/log/? + options: + - + - Yes, there are additional clues in /opt/so/log/ (please provide detail below) + - No, there are no additional clues + validations: + required: true + - type: textarea + attributes: + label: Detail + description: Please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and then provide detailed information to help us help you. + placeholder: |- + STOP! Before typing, please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 in their entirety! + + If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support + validations: + required: true + - type: checkboxes + attributes: + label: Guidelines + options: + - label: I have read the discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and assert that I have followed the guidelines. + required: true diff --git a/.github/workflows/close-threads.yml b/.github/workflows/close-threads.yml new file mode 100644 index 000000000..41b267f2b --- /dev/null +++ b/.github/workflows/close-threads.yml @@ -0,0 +1,33 @@ +name: 'Close Threads' + +on: + schedule: + - cron: '50 1 * * *' + workflow_dispatch: + +permissions: + issues: write + pull-requests: write + discussions: write + +concurrency: + group: lock-threads + +jobs: + close-threads: + if: github.repository_owner == 'security-onion-solutions' + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v5 + with: + days-before-issue-stale: -1 + days-before-issue-close: 60 + stale-issue-message: "This issue is stale because it has been inactive for an extended period. Stale issues convey that the issue, while important to someone, is not critical enough for the author, or other community members to work on, sponsor, or otherwise shepherd the issue through to a resolution." + close-issue-message: "This issue was closed because it has been stale for an extended period. It will be automatically locked in 30 days, after which no further commenting will be available." + days-before-pr-stale: 45 + days-before-pr-close: 60 + stale-pr-message: "This PR is stale because it has been inactive for an extended period. The longer a PR remains stale the more out of date with the main branch it becomes." + close-pr-message: "This PR was closed because it has been stale for an extended period. It will be automatically locked in 30 days. If there is still a commitment to finishing this PR re-open it before it is locked." diff --git a/.github/workflows/lock-threads.yml b/.github/workflows/lock-threads.yml new file mode 100644 index 000000000..f5d46ed46 --- /dev/null +++ b/.github/workflows/lock-threads.yml @@ -0,0 +1,26 @@ +name: 'Lock Threads' + +on: + schedule: + - cron: '50 2 * * *' + workflow_dispatch: + +permissions: + issues: write + pull-requests: write + discussions: write + +concurrency: + group: lock-threads + +jobs: + lock-threads: + if: github.repository_owner == 'security-onion-solutions' + runs-on: ubuntu-latest + steps: + - uses: jertel/lock-threads@main + with: + include-discussion-currently-open: true + discussion-inactive-days: 90 + issue-inactive-days: 30 + pr-inactive-days: 30 diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 16641e4cb..9ef2f4665 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,17 +1,17 @@ -### 2.4.40-20240116 ISO image released on 2024/01/17 +### 2.4.80-20240624 ISO image released on 2024/06/25 ### Download and Verify -2.4.40-20240116 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.4.40-20240116.iso +2.4.80-20240624 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso -MD5: AC55D027B663F3CE0878FEBDAD9DD78B -SHA1: C2B51723B17F3DC843CC493EB80E93B123E3A3E1 -SHA256: C5F135FCF45A836BBFF58C231F95E1EA0CD894898322187AD5FBFCD24BC2F123 +MD5: 139F9762E926F9CB3C4A9528A3752C31 +SHA1: BC6CA2C5F4ABC1A04E83A5CF8FFA6A53B1583CC9 +SHA256: 70E90845C84FFA30AD6CF21504634F57C273E7996CA72F7250428DDBAAC5B1BD Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.40-20240116.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS @@ -25,27 +25,29 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2. Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.40-20240116.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.4.40-20240116.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.4.40-20240116.iso.sig securityonion-2.4.40-20240116.iso +gpg --verify securityonion-2.4.80-20240624.iso.sig securityonion-2.4.80-20240624.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Tue 16 Jan 2024 07:34:40 PM EST using RSA key ID FE507013 +gpg: Signature made Mon 24 Jun 2024 02:42:03 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. Primary key fingerprint: C804 A93D 36BE 0C73 3EA1 9644 7C10 60B7 FE50 7013 ``` +If it fails to verify, try downloading again. If it still fails to verify, try downloading from another computer or another network. + Once you've verified the ISO image, you're ready to proceed to our Installation guide: https://docs.securityonion.net/en/2.4/installation.html diff --git a/README.md b/README.md index 19a560419..530a21813 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,22 @@ Alerts ![Alerts](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/50_alerts.png) Dashboards -![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/51_dashboards.png) +![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_dashboards.png) Hunt -![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/52_hunt.png) +![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/56_hunt.png) + +Detections +![Detections](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_detections.png) PCAP -![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_pcap.png) +![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/62_pcap.png) Grid -![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_grid.png) +![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/75_grid.png) Config -![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/61_config.png) +![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/87_config.png) ### Release Notes diff --git a/files/firewall/assigned_hostgroups.local.map.yaml b/files/firewall/assigned_hostgroups.local.map.yaml index be34cb0be..025b32131 100644 --- a/files/firewall/assigned_hostgroups.local.map.yaml +++ b/files/firewall/assigned_hostgroups.local.map.yaml @@ -19,4 +19,4 @@ role: receiver: standalone: searchnode: - sensor: + sensor: \ No newline at end of file diff --git a/files/salt/master/master b/files/salt/master/master index b93fa93de..2333a50aa 100644 --- a/files/salt/master/master +++ b/files/salt/master/master @@ -41,7 +41,8 @@ file_roots: base: - /opt/so/saltstack/local/salt - /opt/so/saltstack/default/salt - + - /nsm/elastic-fleet/artifacts + - /opt/so/rules/nids # The master_roots setting configures a master-only copy of the file_roots dictionary, # used by the state compiler. diff --git a/pillar/kafka/nodes.sls b/pillar/kafka/nodes.sls new file mode 100644 index 000000000..ba14c219e --- /dev/null +++ b/pillar/kafka/nodes.sls @@ -0,0 +1,2 @@ +kafka: + nodes: \ No newline at end of file diff --git a/pillar/top.sls b/pillar/top.sls index 9af62aa0b..f7ec39957 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -43,8 +43,6 @@ base: - soc.soc_soc - soc.adv_soc - soc.license - - soctopus.soc_soctopus - - soctopus.adv_soctopus - kibana.soc_kibana - kibana.adv_kibana - kratos.soc_kratos @@ -61,10 +59,11 @@ base: - elastalert.adv_elastalert - backup.soc_backup - backup.adv_backup - - soctopus.soc_soctopus - - soctopus.adv_soctopus - minions.{{ grains.id }} - minions.adv_{{ grains.id }} + - kafka.nodes + - kafka.soc_kafka + - kafka.adv_kafka - stig.soc_stig '*_sensor': @@ -108,8 +107,6 @@ base: - soc.soc_soc - soc.adv_soc - soc.license - - soctopus.soc_soctopus - - soctopus.adv_soctopus - kibana.soc_kibana - kibana.adv_kibana - strelka.soc_strelka @@ -165,8 +162,6 @@ base: - soc.soc_soc - soc.adv_soc - soc.license - - soctopus.soc_soctopus - - soctopus.adv_soctopus - kibana.soc_kibana - kibana.adv_kibana - strelka.soc_strelka @@ -184,6 +179,9 @@ base: - minions.{{ grains.id }} - minions.adv_{{ grains.id }} - stig.soc_stig + - kafka.nodes + - kafka.soc_kafka + - kafka.adv_kafka '*_heavynode': - elasticsearch.auth @@ -228,6 +226,7 @@ base: - minions.adv_{{ grains.id }} - stig.soc_stig - soc.license + - kafka.nodes '*_receiver': - logstash.nodes @@ -240,6 +239,10 @@ base: - redis.adv_redis - minions.{{ grains.id }} - minions.adv_{{ grains.id }} + - kafka.nodes + - kafka.soc_kafka + - kafka.adv_kafka + - soc.license '*_import': - secrets @@ -262,8 +265,6 @@ base: - soc.soc_soc - soc.adv_soc - soc.license - - soctopus.soc_soctopus - - soctopus.adv_soctopus - kibana.soc_kibana - kibana.adv_kibana - backup.soc_backup diff --git a/pyci.sh b/pyci.sh index e85287063..8cbee5e75 100755 --- a/pyci.sh +++ b/pyci.sh @@ -15,12 +15,16 @@ TARGET_DIR=${1:-.} PATH=$PATH:/usr/local/bin -if ! which pytest &> /dev/null || ! which flake8 &> /dev/null ; then - echo "Missing dependencies. Consider running the following command:" - echo " python -m pip install flake8 pytest pytest-cov" +if [ ! -d .venv ]; then + python -m venv .venv +fi + +source .venv/bin/activate + +if ! pip install flake8 pytest pytest-cov pyyaml; then + echo "Unable to install dependencies." exit 1 fi -pip install pytest pytest-cov flake8 "$TARGET_DIR" "--config=${HOME_DIR}/pytest.ini" -python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100 "$TARGET_DIR" \ No newline at end of file +python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100 "$TARGET_DIR" diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index d27f51ede..eb73e6e29 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -34,7 +34,6 @@ 'suricata', 'utility', 'schedule', - 'soctopus', 'tcpreplay', 'docker_clean' ], @@ -66,6 +65,7 @@ 'registry', 'manager', 'nginx', + 'strelka.manager', 'soc', 'kratos', 'influxdb', @@ -92,6 +92,7 @@ 'nginx', 'telegraf', 'influxdb', + 'strelka.manager', 'soc', 'kratos', 'elasticfleet', @@ -101,9 +102,9 @@ 'suricata.manager', 'utility', 'schedule', - 'soctopus', 'docker_clean', - 'stig' + 'stig', + 'kafka' ], 'so-managersearch': [ 'salt.master', @@ -113,6 +114,7 @@ 'nginx', 'telegraf', 'influxdb', + 'strelka.manager', 'soc', 'kratos', 'elastic-fleet-package-registry', @@ -123,9 +125,9 @@ 'suricata.manager', 'utility', 'schedule', - 'soctopus', 'docker_clean', - 'stig' + 'stig', + 'kafka' ], 'so-searchnode': [ 'ssl', @@ -157,10 +159,10 @@ 'healthcheck', 'utility', 'schedule', - 'soctopus', 'tcpreplay', 'docker_clean', - 'stig' + 'stig', + 'kafka' ], 'so-sensor': [ 'ssl', @@ -180,6 +182,7 @@ 'telegraf', 'firewall', 'logstash', + 'nginx', 'healthcheck', 'schedule', 'elasticfleet', @@ -190,7 +193,10 @@ 'telegraf', 'firewall', 'schedule', - 'docker_clean' + 'docker_clean', + 'kafka', + 'elasticsearch.ca', + 'stig' ], 'so-desktop': [ 'ssl', @@ -199,10 +205,6 @@ ], }, grain='role') %} - {% if grains.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone'] %} - {% do allowed_states.append('mysql') %} - {% endif %} - {%- if grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %} {% do allowed_states.append('zeek') %} {%- endif %} @@ -228,10 +230,6 @@ {% do allowed_states.append('elastalert') %} {% endif %} - {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %} - {% do allowed_states.append('playbook') %} - {% endif %} - {% if grains.role in ['so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-receiver'] %} {% do allowed_states.append('logstash') %} {% endif %} diff --git a/salt/bpf/pcap.map.jinja b/salt/bpf/pcap.map.jinja index c1d7562cc..4d8fef460 100644 --- a/salt/bpf/pcap.map.jinja +++ b/salt/bpf/pcap.map.jinja @@ -1,7 +1,10 @@ -{% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %} -{% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %} -{% import 'bpf/macros.jinja' as MACROS %} - -{{ MACROS.remove_comments(BPFMERGED, 'pcap') }} - -{% set PCAPBPF = BPFMERGED.pcap %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% if GLOBALS.pcap_engine == "TRANSITION" %} +{% set PCAPBPF = ["ip and host 255.255.255.1 and port 1"] %} +{% else %} +{% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %} +{% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %} +{% import 'bpf/macros.jinja' as MACROS %} +{{ MACROS.remove_comments(BPFMERGED, 'pcap') }} +{% set PCAPBPF = BPFMERGED.pcap %} +{% endif %} diff --git a/salt/bpf/soc_bpf.yaml b/salt/bpf/soc_bpf.yaml index 379eaa022..d93ec98fd 100644 --- a/salt/bpf/soc_bpf.yaml +++ b/salt/bpf/soc_bpf.yaml @@ -1,6 +1,6 @@ bpf: pcap: - description: List of BPF filters to apply to PCAP. + description: List of BPF filters to apply to Stenographer. multiline: True forcedType: "[]string" helpLink: bpf.html diff --git a/salt/ca/files/signing_policies.conf b/salt/ca/files/signing_policies.conf index 6f1b1f172..4fc04aacc 100644 --- a/salt/ca/files/signing_policies.conf +++ b/salt/ca/files/signing_policies.conf @@ -1,6 +1,3 @@ -mine_functions: - x509.get_pem_entries: [/etc/pki/ca.crt] - x509_signing_policies: filebeat: - minions: '*' @@ -70,3 +67,17 @@ x509_signing_policies: - authorityKeyIdentifier: keyid,issuer:always - days_valid: 820 - copypath: /etc/pki/issued_certs/ + kafka: + - minions: '*' + - signing_private_key: /etc/pki/ca.key + - signing_cert: /etc/pki/ca.crt + - C: US + - ST: Utah + - L: Salt Lake City + - basicConstraints: "critical CA:false" + - keyUsage: "digitalSignature, keyEncipherment" + - subjectKeyIdentifier: hash + - authorityKeyIdentifier: keyid,issuer:always + - extendedKeyUsage: "serverAuth, clientAuth" + - days_valid: 820 + - copypath: /etc/pki/issued_certs/ diff --git a/salt/common/init.sls b/salt/common/init.sls index 5f13c3893..51836daf6 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -4,7 +4,6 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} include: - - common.soup_scripts - common.packages {% if GLOBALS.role in GLOBALS.manager_roles %} - manager.elasticsearch # needed for elastic_curl_config state @@ -134,6 +133,18 @@ common_sbin_jinja: - file_mode: 755 - template: jinja +{% if not GLOBALS.is_manager%} +# prior to 2.4.50 these scripts were in common/tools/sbin on the manager because of soup and distributed to non managers +# these two states remove the scripts from non manager nodes +remove_soup: + file.absent: + - name: /usr/sbin/soup + +remove_so-firewall: + file.absent: + - name: /usr/sbin/so-firewall +{% endif %} + so-status_script: file.managed: - name: /usr/sbin/so-status diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index 041649200..508b95b67 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -1,23 +1,117 @@ -# Sync some Utilities -soup_scripts: - file.recurse: - - name: /usr/sbin - - user: root - - group: root - - file_mode: 755 - - source: salt://common/tools/sbin - - include_pat: - - so-common - - so-image-common +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. -soup_manager_scripts: - file.recurse: - - name: /usr/sbin - - user: root - - group: root - - file_mode: 755 - - source: salt://manager/tools/sbin - - include_pat: - - so-firewall - - so-repo-sync - - soup +{% if '2.4' in salt['cp.get_file_str']('/etc/soversion') %} + +{% import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %} +{% if SOC_GLOBAL.global.airgap %} +{% set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %} +{% else %} +{% set UPDATE_DIR='/tmp/sogh/securityonion' %} +{% endif %} + +remove_common_soup: + file.absent: + - name: /opt/so/saltstack/default/salt/common/tools/sbin/soup + +remove_common_so-firewall: + file.absent: + - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall + +# This section is used to put the scripts in place in the Salt file system +# in case a state run tries to overwrite what we do in the next section. +copy_so-common_common_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common + - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common + - force: True + - preserve: True + +copy_so-image-common_common_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common + - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common + - force: True + - preserve: True + +copy_soup_manager_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/manager/tools/sbin/soup + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup + - force: True + - preserve: True + +copy_so-firewall_manager_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-firewall + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall + - force: True + - preserve: True + +copy_so-yaml_manager_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-yaml.py + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py + - force: True + - preserve: True + +copy_so-repo-sync_manager_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-repo-sync + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync + - preserve: True + +# This section is used to put the new script in place so that it can be called during soup. +# It is faster than calling the states that normally manage them to put them in place. +copy_so-common_sbin: + file.copy: + - name: /usr/sbin/so-common + - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common + - force: True + - preserve: True + +copy_so-image-common_sbin: + file.copy: + - name: /usr/sbin/so-image-common + - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common + - force: True + - preserve: True + +copy_soup_sbin: + file.copy: + - name: /usr/sbin/soup + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup + - force: True + - preserve: True + +copy_so-firewall_sbin: + file.copy: + - name: /usr/sbin/so-firewall + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall + - force: True + - preserve: True + +copy_so-yaml_sbin: + file.copy: + - name: /usr/sbin/so-yaml.py + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py + - force: True + - preserve: True + +copy_so-repo-sync_sbin: + file.copy: + - name: /usr/sbin/so-repo-sync + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync + - force: True + - preserve: True + +{% else %} +fix_23_soup_sbin: + cmd.run: + - name: curl -s -f -o /usr/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup +fix_23_soup_salt: + cmd.run: + - name: curl -s -f -o /opt/so/saltstack/defalt/salt/common/tools/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup +{% endif %} diff --git a/salt/common/tools/sbin/so-checkin b/salt/common/tools/sbin/so-checkin index db35af410..13ea9acd9 100755 --- a/salt/common/tools/sbin/so-checkin +++ b/salt/common/tools/sbin/so-checkin @@ -5,8 +5,13 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - - . /usr/sbin/so-common -salt-call state.highstate -l info +cat << EOF + +so-checkin will run a full salt highstate to apply all salt states. If a highstate is already running, this request will be queued and so it may pause for a few minutes before you see any more output. For more information about so-checkin and salt, please see: +https://docs.securityonion.net/en/2.4/salt.html + +EOF + +salt-call state.highstate -l info queue=True diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index e09d2c8ae..02d16ad2c 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -8,7 +8,7 @@ # Elastic agent is not managed by salt. Because of this we must store this base information in a # script that accompanies the soup system. Since so-common is one of those special soup files, # and since this same logic is required during installation, it's included in this file. -ELASTIC_AGENT_TARBALL_VERSION="8.10.4" +ELASTIC_AGENT_TARBALL_VERSION="8.14.1" ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" @@ -31,6 +31,11 @@ if ! echo "$PATH" | grep -q "/usr/sbin"; then export PATH="$PATH:/usr/sbin" fi +# See if a proxy is set. If so use it. +if [ -f /etc/profile.d/so-proxy.sh ]; then + . /etc/profile.d/so-proxy.sh +fi + # Define a banner to separate sections banner="=========================================================================" @@ -179,6 +184,21 @@ copy_new_files() { cd /tmp } +create_local_directories() { + echo "Creating local pillar and salt directories if needed" + PILLARSALTDIR=$1 + local_salt_dir="/opt/so/saltstack/local" + for i in "pillar" "salt"; do + for d in $(find $PILLARSALTDIR/$i -type d); do + suffixdir=${d//$PILLARSALTDIR/} + if [ ! -d "$local_salt_dir/$suffixdir" ]; then + mkdir -pv $local_salt_dir$suffixdir + fi + done + chown -R socore:socore $local_salt_dir/$i + done +} + disable_fastestmirror() { sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf } @@ -248,6 +268,14 @@ get_random_value() { head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1 } +get_agent_count() { + if [ -f /opt/so/log/agents/agentstatus.log ]; then + AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}') + else + AGENTCOUNT=0 + fi +} + gpg_rpm_import() { if [[ $is_oracle ]]; then if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then @@ -329,7 +357,7 @@ lookup_salt_value() { local="" fi - salt-call --no-color ${kind}.get ${group}${key} --out=${output} ${local} + salt-call -lerror --no-color ${kind}.get ${group}${key} --out=${output} ${local} } lookup_pillar() { @@ -366,6 +394,13 @@ is_feature_enabled() { return 1 } +read_feat() { + if [ -f /opt/so/log/sostatus/lks_enabled ]; then + lic_id=$(cat /opt/so/saltstack/local/pillar/soc/license.sls | grep license_id: | awk '{print $2}') + echo "$lic_id/$(cat /opt/so/log/sostatus/lks_enabled)/$(cat /opt/so/log/sostatus/fps_enabled)" + fi +} + require_manager() { if is_manager_node; then echo "This is a manager, so we can proceed." @@ -559,6 +594,15 @@ status () { printf "\n=========================================================================\n$(date) | $1\n=========================================================================\n" } +sync_options() { + set_version + set_os + salt_minion_count + get_agent_count + + echo "$VERSION/$OS/$(uname -r)/$MINIONCOUNT:$AGENTCOUNT/$(read_feat)" +} + systemctl_func() { local action=$1 local echo_action=$1 diff --git a/salt/common/tools/sbin/so-common-status-check b/salt/common/tools/sbin/so-common-status-check index b073eb457..4aa981123 100644 --- a/salt/common/tools/sbin/so-common-status-check +++ b/salt/common/tools/sbin/so-common-status-check @@ -37,23 +37,32 @@ def check_needs_restarted(): with open(outfile, 'w') as f: f.write(val) -def check_for_fips(): - fips = 0 +def check_for_fps(): + feat = 'fps' + feat_full = feat.replace('ps', 'ips') + fps = 0 try: - result = subprocess.run(['fips-mode-setup', '--is-enabled'], check=True, stdout=subprocess.PIPE) - fips = int(result.returncode == 0) + result = subprocess.run([feat_full + '-mode-setup', '--is-enabled'], stdout=subprocess.PIPE) + if result.returncode == 0: + fps = 1 except FileNotFoundError: - with open('/proc/sys/crypto/fips_enabled', 'r') as f: - contents = f.read() - if '1' in contents: - fips = 1 - else: - fips = 0 - with open('/opt/so/log/sostatus/fips_enabled', 'w') as f: - f.write(str(fips)) + fn = '/proc/sys/crypto/' + feat_full + '_enabled' + try: + with open(fn, 'r') as f: + contents = f.read() + if '1' in contents: + fps = 1 + except: + # Unknown, so assume 0 + fps = 0 -def check_for_luks(): - luks = 0 + with open('/opt/so/log/sostatus/fps_enabled', 'w') as f: + f.write(str(fps)) + +def check_for_lks(): + feat = 'Lks' + feat_full = feat.replace('ks', 'uks') + lks = 0 result = subprocess.run(['lsblk', '-p', '-J'], check=True, stdout=subprocess.PIPE) data = json.loads(result.stdout) for device in data['blockdevices']: @@ -61,16 +70,18 @@ def check_for_luks(): for gc in device['children']: if 'children' in gc: try: - result = subprocess.run(['cryptsetup', 'isLuks', gc['name']], check=True, stdout=subprocess.PIPE) - luks = int(result.returncode == 0) + arg = 'is' + feat_full + result = subprocess.run(['cryptsetup', arg, gc['name']], stdout=subprocess.PIPE) + if result.returncode == 0: + lks = 1 except FileNotFoundError: for ggc in gc['children']: if 'crypt' in ggc['type']: - luks = 1 - if luks: + lks = 1 + if lks: break - with open('/opt/so/log/sostatus/luks_enabled', 'w') as f: - f.write(str(luks)) + with open('/opt/so/log/sostatus/lks_enabled', 'w') as f: + f.write(str(lks)) def fail(msg): print(msg, file=sys.stderr) @@ -83,9 +94,9 @@ def main(): # Ensure that umask is 0022 so that files created by this script have rw-r-r permissions org_umask = os.umask(0o022) check_needs_restarted() - check_for_fips() - check_for_luks() - # Restore umask to whatever value was set before this script was run. STIG sets to 0077 rw--- + check_for_fps() + check_for_lks() + # Restore umask to whatever value was set before this script was run. SXIG sets to 0077 rw--- os.umask(org_umask) if __name__ == "__main__": diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 7900b3c52..03051cb5f 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -50,16 +50,14 @@ container_list() { "so-idh" "so-idstools" "so-influxdb" + "so-kafka" "so-kibana" "so-kratos" "so-logstash" - "so-mysql" "so-nginx" "so-pcaptools" - "so-playbook" "so-redis" "so-soc" - "so-soctopus" "so-steno" "so-strelka-backend" "so-strelka-filestream" @@ -67,7 +65,7 @@ container_list() { "so-strelka-manager" "so-suricata" "so-telegraf" - "so-zeek" + "so-zeek" ) else TRUSTED_CONTAINERS=( diff --git a/salt/common/tools/sbin/so-ip-update b/salt/common/tools/sbin/so-ip-update index 7278afb94..6a330f644 100755 --- a/salt/common/tools/sbin/so-ip-update +++ b/salt/common/tools/sbin/so-ip-update @@ -49,10 +49,6 @@ if [ "$CONTINUE" == "y" ]; then sed -i "s|$OLD_IP|$NEW_IP|g" $file done - echo "Granting MySQL root user permissions on $NEW_IP" - docker exec -i so-mysql mysql --user=root --password=$(lookup_pillar_secret 'mysql') -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'$NEW_IP' IDENTIFIED BY '$(lookup_pillar_secret 'mysql')' WITH GRANT OPTION;" &> /dev/null - echo "Removing MySQL root user from $OLD_IP" - docker exec -i so-mysql mysql --user=root --password=$(lookup_pillar_secret 'mysql') -e "DROP USER 'root'@'$OLD_IP';" &> /dev/null echo "Updating Kibana dashboards" salt-call state.apply kibana.so_savedobjects_defaults -l info queue=True diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index b8c68ffa1..cf1691589 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -122,6 +122,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error while communicating" # Elasticsearch MS -> HN "sensor" temporarily unavailable EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tls handshake error" # Docker registry container when new node comes onlines EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to get license information" # Logstash trying to contact ES before it's ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process already finished" # Telegraf script finished just as the auto kill timeout kicked in fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then @@ -154,15 +155,11 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fail\\(error\\)" # redis/python generic stack line, rely on other lines for actual error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|urlerror" # idstools connection timeout EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeouterror" # idstools connection timeout - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|forbidden" # playbook EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml" # Elastic ML errors EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context canceled" # elastic agent during shutdown - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|exited with code 128" # soctopus errors during forced restart by highstate EXCLUDED_ERRORS="$EXCLUDED_ERRORS|geoip databases update" # airgap can't update GeoIP DB EXCLUDED_ERRORS="$EXCLUDED_ERRORS|filenotfounderror" # bug in 2.4.10 filecheck salt state caused duplicate cronjobs EXCLUDED_ERRORS="$EXCLUDED_ERRORS|salt-minion-check" # bug in early 2.4 place Jinja script in non-jinja salt dir causing cron output errors - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|generating elastalert config" # playbook expected error - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|activerecord" # playbook expected error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|monitoring.metrics" # known issue with elastic agent casting the field incorrectly if an integer value shows up before a float EXCLUDED_ERRORS="$EXCLUDED_ERRORS|repodownload.conf" # known issue with reposync on pre-2.4.20 EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing versions record" # stenographer corrupt index @@ -201,7 +198,13 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|req.LocalMeta.host.ip" # known issue in GH EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sendmail" # zeek EXCLUDED_ERRORS="$EXCLUDED_ERRORS|stats.log" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unknown column" # Elastalert errors from running EQL queries + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parsing_exception" # Elastalert EQL parsing issue. Temp. EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context deadline exceeded" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error running query:" # Specific issues with detection rules + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|detect-parse" # Suricata encountering a malformed rule + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed" # Detections: Exclude false positive due to automated testing + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors" # Detections: Not an actual error fi RESULT=0 @@ -210,7 +213,9 @@ RESULT=0 CONTAINER_IDS=$(docker ps -q) exclude_container so-kibana # kibana error logs are too verbose with large varieties of errors most of which are temporary exclude_container so-idstools # ignore due to known issues and noisy logging -exclude_container so-playbook # ignore due to several playbook known issues +exclude_container so-playbook # Playbook is removed as of 2.4.70, disregard output in stopped containers +exclude_container so-mysql # MySQL is removed as of 2.4.70, disregard output in stopped containers +exclude_container so-soctopus # Soctopus is removed as of 2.4.70, disregard output in stopped containers for container_id in $CONTAINER_IDS; do container_name=$(docker ps --format json | jq ". | select(.ID==\"$container_id\")|.Names") @@ -228,10 +233,14 @@ exclude_log "kibana.log" # kibana error logs are too verbose with large variet exclude_log "spool" # disregard zeek analyze logs as this is data specific exclude_log "import" # disregard imported test data the contains error strings exclude_log "update.log" # ignore playbook updates due to several known issues -exclude_log "playbook.log" # ignore due to several playbook known issues exclude_log "cron-cluster-delete.log" # ignore since Curator has been removed exclude_log "cron-close.log" # ignore since Curator has been removed -exclude_log "curator.log" # ignore since Curator has been removed +exclude_log "curator.log" # ignore since Curator has been removed +exclude_log "playbook.log" # Playbook is removed as of 2.4.70, logs may still be on disk +exclude_log "mysqld.log" # MySQL is removed as of 2.4.70, logs may still be on disk +exclude_log "soctopus.log" # Soctopus is removed as of 2.4.70, logs may still be on disk +exclude_log "agentstatus.log" # ignore this log since it tracks agents in error state +exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable for log_file in $(cat /tmp/log_check_files); do status "Checking log file $log_file" diff --git a/salt/common/tools/sbin/so-luks-tpm-regen b/salt/common/tools/sbin/so-luks-tpm-regen new file mode 100644 index 000000000..50058b504 --- /dev/null +++ b/salt/common/tools/sbin/so-luks-tpm-regen @@ -0,0 +1,98 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0." + +set -e +# This script is intended to be used in the case the ISO install did not properly setup TPM decrypt for LUKS partitions at boot. +if [ -z $NOROOT ]; then + # Check for prerequisites + if [ "$(id -u)" -ne 0 ]; then + echo "This script must be run using sudo!" + exit 1 + fi +fi +ENROLL_TPM=N + +while [[ $# -gt 0 ]]; do + case $1 in + --enroll-tpm) + ENROLL_TPM=Y + ;; + *) + echo "Usage: $0 [options]" + echo "" + echo "where options are:" + echo " --enroll-tpm for when TPM enrollment was not selected during ISO install." + echo "" + exit 1 + ;; + esac + shift +done + +check_for_tpm() { + echo -n "Checking for TPM: " + if [ -d /sys/class/tpm/tpm0 ]; then + echo -e "tpm0 found." + TPM="yes" + # Check if TPM is using sha1 or sha256 + if [ -d /sys/class/tpm/tpm0/pcr-sha1 ]; then + echo -e "TPM is using sha1.\n" + TPM_PCR="sha1" + elif [ -d /sys/class/tpm/tpm0/pcr-sha256 ]; then + echo -e "TPM is using sha256.\n" + TPM_PCR="sha256" + fi + else + echo -e "No TPM found.\n" + exit 1 + fi +} + +check_for_luks_partitions() { + echo "Checking for LUKS partitions" + for part in $(lsblk -o NAME,FSTYPE -ln | grep crypto_LUKS | awk '{print $1}'); do + echo "Found LUKS partition: $part" + LUKS_PARTITIONS+=("$part") + done + if [ ${#LUKS_PARTITIONS[@]} -eq 0 ]; then + echo -e "No LUKS partitions found.\n" + exit 1 + fi + echo "" +} + +enroll_tpm_in_luks() { + read -s -p "Enter the LUKS passphrase used during ISO install: " LUKS_PASSPHRASE + echo "" + for part in "${LUKS_PARTITIONS[@]}"; do + echo "Enrolling TPM for LUKS device: /dev/$part" + if [ "$TPM_PCR" == "sha1" ]; then + clevis luks bind -d /dev/$part tpm2 '{"pcr_bank":"sha1","pcr_ids":"7"}' <<< $LUKS_PASSPHRASE + elif [ "$TPM_PCR" == "sha256" ]; then + clevis luks bind -d /dev/$part tpm2 '{"pcr_bank":"sha256","pcr_ids":"7"}' <<< $LUKS_PASSPHRASE + fi + done + } + +regenerate_tpm_enrollment_token() { + for part in "${LUKS_PARTITIONS[@]}"; do + clevis luks regen -d /dev/$part -s 1 -q + done +} + +check_for_tpm +check_for_luks_partitions + +if [[ $ENROLL_TPM == "Y" ]]; then + enroll_tpm_in_luks +else + regenerate_tpm_enrollment_token +fi + +echo "Running dracut" +dracut -fv +echo -e "\nTPM configuration complete. Reboot the system to verify the TPM is correctly decrypting the LUKS partition(s) at boot.\n" \ No newline at end of file diff --git a/salt/common/tools/sbin_jinja/so-import-pcap b/salt/common/tools/sbin_jinja/so-import-pcap index b8a90421f..d3886305e 100755 --- a/salt/common/tools/sbin_jinja/so-import-pcap +++ b/salt/common/tools/sbin_jinja/so-import-pcap @@ -89,6 +89,7 @@ function suricata() { -v ${LOG_PATH}:/var/log/suricata/:rw \ -v ${NSM_PATH}/:/nsm/:rw \ -v "$PCAP:/input.pcap:ro" \ + -v /dev/null:/nsm/suripcap:rw \ -v /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro \ {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-suricata:{{ VERSION }} \ --runmode single -k none -r /input.pcap > $LOG_PATH/console.log 2>&1 @@ -247,7 +248,7 @@ fi START_OLDEST_SLASH=$(echo $START_OLDEST | sed -e 's/-/%2F/g') END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g') if [[ $VALID_PCAPS_COUNT -gt 0 ]] || [[ $SKIPPED_PCAPS_COUNT -gt 0 ]]; then - URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20-sankey%20event.dataset%20event.category%2a%20%7C%20groupby%20-pie%20event.category%20%7C%20groupby%20-bar%20event.module%20%7C%20groupby%20event.dataset%20%7C%20groupby%20event.module%20%7C%20groupby%20event.category%20%7C%20groupby%20observer.name%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC" + URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20event.module*%20%7C%20groupby%20-sankey%20event.module*%20event.dataset%20%7C%20groupby%20event.dataset%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20network.protocol%20%7C%20groupby%20rule.name%20rule.category%20event.severity_label%20%7C%20groupby%20dns.query.name%20%7C%20groupby%20file.mime_type%20%7C%20groupby%20http.virtual_host%20http.uri%20%7C%20groupby%20notice.note%20notice.message%20notice.sub_message%20%7C%20groupby%20ssl.server_name%20%7C%20groupby%20source_geo.organization_name%20source.geo.country_name%20%7C%20groupby%20destination_geo.organization_name%20destination.geo.country_name&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC" status "Import complete!" status diff --git a/salt/common/tools/sbin/so-tcpreplay b/salt/common/tools/sbin_jinja/so-tcpreplay similarity index 96% rename from salt/common/tools/sbin/so-tcpreplay rename to salt/common/tools/sbin_jinja/so-tcpreplay index 99314c289..969ca699f 100755 --- a/salt/common/tools/sbin/so-tcpreplay +++ b/salt/common/tools/sbin_jinja/so-tcpreplay @@ -10,7 +10,7 @@ . /usr/sbin/so-common . /usr/sbin/so-image-common -REPLAYIFACE=${REPLAYIFACE:-$(lookup_pillar interface sensor)} +REPLAYIFACE=${REPLAYIFACE:-"{{salt['pillar.get']('sensor:interface', '')}}"} REPLAYSPEED=${REPLAYSPEED:-10} mkdir -p /opt/so/samples diff --git a/salt/desktop/packages.sls b/salt/desktop/packages.sls index 9a7d53317..b2a028e60 100644 --- a/salt/desktop/packages.sls +++ b/salt/desktop/packages.sls @@ -334,6 +334,7 @@ desktop_packages: - pulseaudio-libs - pulseaudio-libs-glib2 - pulseaudio-utils + - putty - sane-airscan - sane-backends - sane-backends-drivers-cameras diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 9a27843ae..161dde485 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -67,13 +67,6 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] - 'so-mysql': - final_octet: 30 - port_bindings: - - 0.0.0.0:3306:3306 - custom_bind_mounts: [] - extra_hosts: [] - extra_env: [] 'so-nginx': final_octet: 31 port_bindings: @@ -84,10 +77,10 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] - 'so-playbook': - final_octet: 32 + 'so-nginx-fleet-node': + final_octet: 31 port_bindings: - - 0.0.0.0:3000:3000 + - 8443:8443 custom_bind_mounts: [] extra_hosts: [] extra_env: [] @@ -111,13 +104,6 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] - 'so-soctopus': - final_octet: 35 - port_bindings: - - 0.0.0.0:7000:7000 - custom_bind_mounts: [] - extra_hosts: [] - extra_env: [] 'so-strelka-backend': final_octet: 36 custom_bind_mounts: [] @@ -194,8 +180,19 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] + ulimits: + - memlock=524288000 'so-zeek': final_octet: 99 custom_bind_mounts: [] extra_hosts: [] extra_env: [] + 'so-kafka': + final_octet: 88 + port_bindings: + - 0.0.0.0:9092:9092 + - 0.0.0.0:9093:9093 + - 0.0.0.0:8778:8778 + custom_bind_mounts: [] + extra_hosts: [] + extra_env: [] diff --git a/salt/docker/init.sls b/salt/docker/init.sls index 769c58af8..1e37364bc 100644 --- a/salt/docker/init.sls +++ b/salt/docker/init.sls @@ -20,30 +20,30 @@ dockergroup: dockerheldpackages: pkg.installed: - pkgs: - - containerd.io: 1.6.21-1 - - docker-ce: 5:24.0.3-1~debian.12~bookworm - - docker-ce-cli: 5:24.0.3-1~debian.12~bookworm - - docker-ce-rootless-extras: 5:24.0.3-1~debian.12~bookworm + - containerd.io: 1.6.33-1 + - docker-ce: 5:26.1.4-1~debian.12~bookworm + - docker-ce-cli: 5:26.1.4-1~debian.12~bookworm + - docker-ce-rootless-extras: 5:26.1.4-1~debian.12~bookworm - hold: True - update_holds: True {% elif grains.oscodename == 'jammy' %} dockerheldpackages: pkg.installed: - pkgs: - - containerd.io: 1.6.21-1 - - docker-ce: 5:24.0.2-1~ubuntu.22.04~jammy - - docker-ce-cli: 5:24.0.2-1~ubuntu.22.04~jammy - - docker-ce-rootless-extras: 5:24.0.2-1~ubuntu.22.04~jammy + - containerd.io: 1.6.33-1 + - docker-ce: 5:26.1.4-1~ubuntu.22.04~jammy + - docker-ce-cli: 5:26.1.4-1~ubuntu.22.04~jammy + - docker-ce-rootless-extras: 5:26.1.4-1~ubuntu.22.04~jammy - hold: True - update_holds: True {% else %} dockerheldpackages: pkg.installed: - pkgs: - - containerd.io: 1.4.9-1 - - docker-ce: 5:20.10.8~3-0~ubuntu-focal - - docker-ce-cli: 5:20.10.5~3-0~ubuntu-focal - - docker-ce-rootless-extras: 5:20.10.5~3-0~ubuntu-focal + - containerd.io: 1.6.33-1 + - docker-ce: 5:26.1.4-1~ubuntu.20.04~focal + - docker-ce-cli: 5:26.1.4-1~ubuntu.20.04~focal + - docker-ce-rootless-extras: 5:26.1.4-1~ubuntu.20.04~focal - hold: True - update_holds: True {% endif %} @@ -51,10 +51,10 @@ dockerheldpackages: dockerheldpackages: pkg.installed: - pkgs: - - containerd.io: 1.6.21-3.1.el9 - - docker-ce: 24.0.4-1.el9 - - docker-ce-cli: 24.0.4-1.el9 - - docker-ce-rootless-extras: 24.0.4-1.el9 + - containerd.io: 1.6.33-3.1.el9 + - docker-ce: 3:26.1.4-1.el9 + - docker-ce-cli: 1:26.1.4-1.el9 + - docker-ce-rootless-extras: 26.1.4-1.el9 - hold: True - update_holds: True {% endif %} diff --git a/salt/docker/soc_docker.yaml b/salt/docker/soc_docker.yaml index 850324a9e..e7ecba6be 100644 --- a/salt/docker/soc_docker.yaml +++ b/salt/docker/soc_docker.yaml @@ -46,13 +46,11 @@ docker: so-kibana: *dockerOptions so-kratos: *dockerOptions so-logstash: *dockerOptions - so-mysql: *dockerOptions so-nginx: *dockerOptions - so-playbook: *dockerOptions + so-nginx-fleet-node: *dockerOptions so-redis: *dockerOptions so-sensoroni: *dockerOptions so-soc: *dockerOptions - so-soctopus: *dockerOptions so-strelka-backend: *dockerOptions so-strelka-filestream: *dockerOptions so-strelka-frontend: *dockerOptions @@ -65,5 +63,42 @@ docker: so-elastic-agent: *dockerOptions so-telegraf: *dockerOptions so-steno: *dockerOptions - so-suricata: *dockerOptions + so-suricata: + final_octet: + description: Last octet of the container IP address. + helpLink: docker.html + readonly: True + advanced: True + global: True + port_bindings: + description: List of port bindings for the container. + helpLink: docker.html + advanced: True + multiline: True + forcedType: "[]string" + custom_bind_mounts: + description: List of custom local volume bindings. + advanced: True + helpLink: docker.html + multiline: True + forcedType: "[]string" + extra_hosts: + description: List of additional host entries for the container. + advanced: True + helpLink: docker.html + multiline: True + forcedType: "[]string" + extra_env: + description: List of additional ENV entries for the container. + advanced: True + helpLink: docker.html + multiline: True + forcedType: "[]string" + ulimits: + description: Ulimits for the container, in bytes. + advanced: True + helpLink: docker.html + multiline: True + forcedType: "[]string" so-zeek: *dockerOptions + so-kafka: *dockerOptions \ No newline at end of file diff --git a/salt/elastalert/config.sls b/salt/elastalert/config.sls index 252aa83c0..25d5bf5f8 100644 --- a/salt/elastalert/config.sls +++ b/salt/elastalert/config.sls @@ -82,6 +82,36 @@ elastasomodulesync: - group: 933 - makedirs: True +elastacustomdir: + file.directory: + - name: /opt/so/conf/elastalert/custom + - user: 933 + - group: 933 + - makedirs: True + +elastacustomsync: + file.recurse: + - name: /opt/so/conf/elastalert/custom + - source: salt://elastalert/files/custom + - user: 933 + - group: 933 + - makedirs: True + - file_mode: 660 + - show_changes: False + +elastapredefinedsync: + file.recurse: + - name: /opt/so/conf/elastalert/predefined + - source: salt://elastalert/files/predefined + - user: 933 + - group: 933 + - makedirs: True + - template: jinja + - file_mode: 660 + - context: + elastalert: {{ ELASTALERTMERGED }} + - show_changes: False + elastaconf: file.managed: - name: /opt/so/conf/elastalert/elastalert_config.yaml diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index a01c80952..393932992 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -1,5 +1,6 @@ elastalert: enabled: False + alerter_parameters: "" config: rules_folder: /opt/elastalert/rules/ scan_subdirectories: true diff --git a/salt/elastalert/enabled.sls b/salt/elastalert/enabled.sls index e4b3642db..6a1ff1440 100644 --- a/salt/elastalert/enabled.sls +++ b/salt/elastalert/enabled.sls @@ -30,6 +30,8 @@ so-elastalert: - /opt/so/rules/elastalert:/opt/elastalert/rules/:ro - /opt/so/log/elastalert:/var/log/elastalert:rw - /opt/so/conf/elastalert/modules/:/opt/elastalert/modules/:ro + - /opt/so/conf/elastalert/predefined/:/opt/elastalert/predefined/:ro + - /opt/so/conf/elastalert/custom/:/opt/elastalert/custom/:ro - /opt/so/conf/elastalert/elastalert_config.yaml:/opt/elastalert/config.yaml:ro {% if DOCKER.containers['so-elastalert'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-elastalert'].custom_bind_mounts %} diff --git a/salt/elastalert/files/custom/placeholder b/salt/elastalert/files/custom/placeholder new file mode 100644 index 000000000..42e4ae4f0 --- /dev/null +++ b/salt/elastalert/files/custom/placeholder @@ -0,0 +1 @@ +THIS IS A PLACEHOLDER FILE \ No newline at end of file diff --git a/salt/elastalert/files/modules/so/playbook-es.py b/salt/elastalert/files/modules/so/playbook-es.py deleted file mode 100644 index 3a43c26c1..000000000 --- a/salt/elastalert/files/modules/so/playbook-es.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -from time import gmtime, strftime -import requests,json -from elastalert.alerts import Alerter - -import urllib3 -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - -class PlaybookESAlerter(Alerter): - """ - Use matched data to create alerts in elasticsearch - """ - - required_options = set(['play_title','play_url','sigma_level']) - - def alert(self, matches): - for match in matches: - today = strftime("%Y.%m.%d", gmtime()) - timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime()) - headers = {"Content-Type": "application/json"} - - creds = None - if 'es_username' in self.rule and 'es_password' in self.rule: - creds = (self.rule['es_username'], self.rule['es_password']) - - payload = {"tags":"alert","rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp} - url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-playbook.alerts-so/_doc/" - requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds) - - def get_info(self): - return {'type': 'PlaybookESAlerter'} diff --git a/salt/elastalert/files/modules/so/securityonion-es.py b/salt/elastalert/files/modules/so/securityonion-es.py new file mode 100644 index 000000000..d9bb8009e --- /dev/null +++ b/salt/elastalert/files/modules/so/securityonion-es.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + +from time import gmtime, strftime +import requests,json +from elastalert.alerts import Alerter + +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +class SecurityOnionESAlerter(Alerter): + """ + Use matched data to create alerts in Elasticsearch. + """ + + required_options = set(['detection_title', 'sigma_level']) + optional_fields = ['sigma_category', 'sigma_product', 'sigma_service'] + + def alert(self, matches): + for match in matches: + timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime()) + headers = {"Content-Type": "application/json"} + + creds = None + if 'es_username' in self.rule and 'es_password' in self.rule: + creds = (self.rule['es_username'], self.rule['es_password']) + + # Start building the rule dict + rule_info = { + "name": self.rule['detection_title'], + "uuid": self.rule['detection_public_id'] + } + + # Add optional fields if they are present in the rule + for field in self.optional_fields: + rule_key = field.split('_')[-1] # Assumes field format "sigma_" + if field in self.rule: + rule_info[rule_key] = self.rule[field] + + # Construct the payload with the conditional rule_info + payload = { + "tags": "alert", + "rule": rule_info, + "event": { + "severity": self.rule['event.severity'], + "module": self.rule['event.module'], + "dataset": self.rule['event.dataset'], + "severity_label": self.rule['sigma_level'] + }, + "sigma_level": self.rule['sigma_level'], + "event_data": match, + "@timestamp": timestamp + } + url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-detections.alerts-so/_doc/" + requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds) + + def get_info(self): + return {'type': 'SecurityOnionESAlerter'} diff --git a/salt/elastalert/files/predefined/jira_auth.yaml b/salt/elastalert/files/predefined/jira_auth.yaml new file mode 100644 index 000000000..9bf0425c0 --- /dev/null +++ b/salt/elastalert/files/predefined/jira_auth.yaml @@ -0,0 +1,6 @@ +{% if elastalert.get('jira_user', '') | length > 0 and elastalert.get('jira_pass', '') | length > 0 %} +user: {{ elastalert.jira_user }} +password: {{ elastalert.jira_pass }} +{% else %} +apikey: {{ elastalert.get('jira_api_key', '') }} +{% endif %} \ No newline at end of file diff --git a/salt/elastalert/files/predefined/smtp_auth.yaml b/salt/elastalert/files/predefined/smtp_auth.yaml new file mode 100644 index 000000000..d09d101f1 --- /dev/null +++ b/salt/elastalert/files/predefined/smtp_auth.yaml @@ -0,0 +1,2 @@ +user: {{ elastalert.get('smtp_user', '') }} +password: {{ elastalert.get('smtp_pass', '') }} diff --git a/salt/elastalert/map.jinja b/salt/elastalert/map.jinja index cc395d8ee..b8ceca277 100644 --- a/salt/elastalert/map.jinja +++ b/salt/elastalert/map.jinja @@ -13,3 +13,19 @@ {% do ELASTALERTDEFAULTS.elastalert.config.update({'es_password': pillar.elasticsearch.auth.users.so_elastic_user.pass}) %} {% set ELASTALERTMERGED = salt['pillar.get']('elastalert', ELASTALERTDEFAULTS.elastalert, merge=True) %} + +{% if 'ntf' in salt['pillar.get']('features', []) %} + {% set params = ELASTALERTMERGED.get('alerter_parameters', '') | load_yaml %} + {% if params != None and params | length > 0 %} + {% do ELASTALERTMERGED.config.update(params) %} + {% endif %} + + {% if ELASTALERTMERGED.get('smtp_user', '') | length > 0 %} + {% do ELASTALERTMERGED.config.update({'smtp_auth_file': '/opt/elastalert/predefined/smtp_auth.yaml'}) %} + {% endif %} + + {% if ELASTALERTMERGED.get('jira_user', '') | length > 0 or ELASTALERTMERGED.get('jira_key', '') | length > 0 %} + {% do ELASTALERTMERGED.config.update({'jira_account_file': '/opt/elastalert/predefined/jira_auth.yaml'}) %} + {% endif %} + +{% endif %} diff --git a/salt/elastalert/soc_elastalert.yaml b/salt/elastalert/soc_elastalert.yaml index cde09b83e..435c5be6a 100644 --- a/salt/elastalert/soc_elastalert.yaml +++ b/salt/elastalert/soc_elastalert.yaml @@ -2,6 +2,99 @@ elastalert: enabled: description: You can enable or disable Elastalert. helpLink: elastalert.html + alerter_parameters: + title: Alerter Parameters + description: Optional configuration parameters for additional alerters that can be enabled for all Sigma rules. Filter for 'Alerter' in this Configuration screen to find the setting that allows these alerters to be enabled within the SOC ElastAlert module. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: elastalert.html + forcedType: string + jira_api_key: + title: Jira API Key + description: Optional configuration parameter for Jira API Key, used instead of the Jira username and password. Requires a valid Security Onion license key. + global: True + sensitive: True + helpLink: elastalert.html + forcedType: string + jira_pass: + title: Jira Password + description: Optional configuration parameter for Jira password. Requires a valid Security Onion license key. + global: True + sensitive: True + helpLink: elastalert.html + forcedType: string + jira_user: + title: Jira Username + description: Optional configuration parameter for Jira username. Requires a valid Security Onion license key. + global: True + helpLink: elastalert.html + forcedType: string + smtp_pass: + title: SMTP Password + description: Optional configuration parameter for SMTP password, required for authenticating email servers. Requires a valid Security Onion license key. + global: True + sensitive: True + helpLink: elastalert.html + forcedType: string + smtp_user: + title: SMTP Username + description: Optional configuration parameter for SMTP username, required for authenticating email servers. Requires a valid Security Onion license key. + global: True + helpLink: elastalert.html + forcedType: string + files: + custom: + alertmanager_ca__crt: + description: Optional custom Certificate Authority for connecting to an AlertManager server. To utilize this custom file, the alertmanager_ca_certs key must be set to /opt/elastalert/custom/alertmanager_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + gelf_ca__crt: + description: Optional custom Certificate Authority for connecting to a Graylog server. To utilize this custom file, the graylog_ca_certs key must be set to /opt/elastalert/custom/graylog_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + http_post_ca__crt: + description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the legacy HTTP POST alerter. To utilize this custom file, the http_post_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + http_post2_ca__crt: + description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the newer HTTP POST 2 alerter. To utilize this custom file, the http_post2_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + ms_teams_ca__crt: + description: Optional custom Certificate Authority for connecting to Microsoft Teams server. To utilize this custom file, the ms_teams_ca_certs key must be set to /opt/elastalert/custom/ms_teams_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + pagerduty_ca__crt: + description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the pagerduty_ca_certs key must be set to /opt/elastalert/custom/pagerduty_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + rocket_chat_ca__crt: + description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the rocket_chart_ca_certs key must be set to /opt/elastalert/custom/rocket_chat_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + smtp__crt: + description: Optional custom certificate for connecting to an SMTP server. To utilize this custom file, the smtp_cert_file key must be set to /opt/elastalert/custom/smtp.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + smtp__key: + description: Optional custom certificate key for connecting to an SMTP server. To utilize this custom file, the smtp_key_file key must be set to /opt/elastalert/custom/smtp.key in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + slack_ca__crt: + description: Optional custom Certificate Authority for connecting to Slack. To utilize this custom file, the slack_ca_certs key must be set to /opt/elastalert/custom/slack_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html config: disable_rules_on_error: description: Disable rules on failure. diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index e4f54ceb0..2d9ab97a1 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -37,6 +37,7 @@ elasticfleet: - azure - barracuda - carbonblack_edr + - cef - checkpoint - cisco_asa - cisco_duo @@ -65,6 +66,7 @@ elasticfleet: - http_endpoint - httpjson - iis + - journald - juniper - juniper_srx - kafka_log @@ -117,3 +119,8 @@ elasticfleet: base_url: https://api.platform.sublimesecurity.com poll_interval: 5m limit: 100 + kismet: + base_url: http://localhost:2501 + poll_interval: 1m + api_key: + enabled_nodes: [] diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index fef85d24c..af5e552eb 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -17,12 +17,19 @@ include: - elasticfleet.sostatus - ssl +# Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready +wait_for_elasticsearch_elasticfleet: + cmd.run: + - name: so-elasticsearch-wait + # If enabled, automatically update Fleet Logstash Outputs {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %} so-elastic-fleet-auto-configure-logstash-outputs: cmd.run: - name: /usr/sbin/so-elastic-fleet-outputs-update - - retry: True + - retry: + attempts: 4 + interval: 30 {% endif %} # If enabled, automatically update Fleet Server URLs & ES Connection @@ -30,15 +37,35 @@ so-elastic-fleet-auto-configure-logstash-outputs: so-elastic-fleet-auto-configure-server-urls: cmd.run: - name: /usr/sbin/so-elastic-fleet-urls-update - - retry: True + - retry: + attempts: 4 + interval: 30 {% endif %} -# Automatically update Fleet Server Elasticsearch URLs +# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs {% if grains.role not in ['so-fleet'] %} so-elastic-fleet-auto-configure-elasticsearch-urls: cmd.run: - name: /usr/sbin/so-elastic-fleet-es-url-update - - retry: True + - retry: + attempts: 4 + interval: 30 + +so-elastic-fleet-auto-configure-artifact-urls: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-artifacts-url-update + - retry: + attempts: 4 + interval: 30 + +{% endif %} + +# Sync Elastic Agent artifacts to Fleet Node +{% if grains.role in ['so-fleet'] %} +elasticagent_syncartifacts: + file.recurse: + - name: /nsm/elastic-fleet/artifacts/beats + - source: salt://beats {% endif %} {% if SERVICETOKEN != '' %} diff --git a/salt/elasticfleet/files/integrations-optional/kismet.json b/salt/elasticfleet/files/integrations-optional/kismet.json new file mode 100644 index 000000000..9a333f31c --- /dev/null +++ b/salt/elasticfleet/files/integrations-optional/kismet.json @@ -0,0 +1,36 @@ +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} +{% raw %} +{ + "package": { + "name": "httpjson", + "version": "" + }, + "name": "kismet-logs", + "namespace": "so", + "description": "Kismet Logs", + "policy_id": "FleetServer_{% endraw %}{{ NAME }}{% raw %}", + "inputs": { + "generic-httpjson": { + "enabled": true, + "streams": { + "httpjson.generic": { + "enabled": true, + "vars": { + "data_stream.dataset": "kismet", + "request_url": "{% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.kismet.base_url }}{% raw %}/devices/last-time/-600/devices.tjson", + "request_interval": "{% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.kismet.poll_interval }}{% raw %}", + "request_method": "GET", + "request_transforms": "- set:\r\n target: header.Cookie\r\n value: 'KISMET={% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.kismet.api_key }}{% raw %}'", + "request_redirect_headers_ban_list": [], + "oauth_scopes": [], + "processors": "", + "tags": [], + "pipeline": "kismet.common" + } + } + } + } + }, + "force": true +} +{% endraw %} \ No newline at end of file diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index de35f803b..15f08a151 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "8.10.2" + "version": "8.14.0" }, "enabled": true, "policy_id": "endpoints-initial", diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json b/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json new file mode 100644 index 000000000..ac4394e62 --- /dev/null +++ b/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "winlog", + "version": "" + }, + "name": "windows-defender", + "namespace": "default", + "description": "Windows Defender - Operational logs", + "policy_id": "endpoints-initial", + "inputs": { + "winlogs-winlog": { + "enabled": true, + "streams": { + "winlog.winlog": { + "enabled": true, + "vars": { + "channel": "Microsoft-Windows-Windows Defender/Operational", + "data_stream.dataset": "winlog.winlog", + "preserve_original_event": false, + "providers": [], + "ignore_older": "72h", + "language": 0, + "tags": [] } + } + } + } + }, + "force": true +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json new file mode 100644 index 000000000..a97faaa5f --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json @@ -0,0 +1,34 @@ +{ + "package": { + "name": "log", + "version": "" + }, + "name": "rita-logs", + "namespace": "so", + "description": "RITA Logs", + "policy_id": "so-grid-nodes_general", + "vars": {}, + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.logs": { + "enabled": true, + "vars": { + "paths": [ + "/nsm/rita/beacons.csv", + "/nsm/rita/exploded-dns.csv", + "/nsm/rita/long-connections.csv" + ], + "exclude_files": [], + "ignore_older": "72h", + "data_stream.dataset": "rita", + "tags": [], + "processors": "- dissect:\n tokenizer: \"/nsm/rita/%{pipeline}.csv\"\n field: \"log.file.path\"\n trim_chars: \".csv\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\").split(\"-\");\n if (pl.length > 1) {\n pl = pl[1];\n }\n else {\n pl = pl[0];\n }\n event.Put(\"@metadata.pipeline\", \"rita.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: rita", + "custom": "exclude_lines: ['^Score', '^Source', '^Domain', '^No results']" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json new file mode 100644 index 000000000..5649b481d --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json @@ -0,0 +1,35 @@ +{ + "policy_id": "so-grid-nodes_general", + "package": { + "name": "log", + "version": "" + }, + "name": "soc-detections-logs", + "description": "Security Onion Console - Detections Logs", + "namespace": "so", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.logs": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/soc/detections_runtime-status_sigma.log", + "/opt/so/log/soc/detections_runtime-status_yara.log" + ], + "exclude_files": [], + "ignore_older": "72h", + "data_stream.dataset": "soc", + "tags": [ + "so-soc" + ], + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"soc\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: detections\n- rename:\n fields:\n - from: \"soc.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"soc.fields.status\"\n to: \"http.response.status_code\"\n - from: \"soc.fields.method\"\n to: \"http.request.method\"\n - from: \"soc.fields.path\"\n to: \"url.path\"\n - from: \"soc.message\"\n to: \"event.action\"\n - from: \"soc.level\"\n to: \"log.level\"\n ignore_missing: true", + "custom": "pipeline: common" + } + } + } + } + }, + "force": true +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json index 8e6bf7958..98204e894 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json @@ -16,6 +16,9 @@ "paths": [ "/var/log/auth.log*", "/var/log/secure*" + ], + "tags": [ + "so-grid-node" ] } }, @@ -25,6 +28,9 @@ "paths": [ "/var/log/messages*", "/var/log/syslog*" + ], + "tags": [ + "so-grid-node" ] } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json index 6c42086bc..42918cc97 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json @@ -16,6 +16,9 @@ "paths": [ "/var/log/auth.log*", "/var/log/secure*" + ], + "tags": [ + "so-grid-node" ] } }, @@ -25,6 +28,9 @@ "paths": [ "/var/log/messages*", "/var/log/syslog*" + ], + "tags": [ + "so-grid-node" ] } } diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 9a0cd0a91..7ed97e6ec 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -79,3 +79,29 @@ elasticfleet: helpLink: elastic-fleet.html advanced: True forcedType: int + kismet: + base_url: + description: Base URL for Kismet. + global: True + helpLink: elastic-fleet.html + advanced: True + forcedType: string + poll_interval: + description: Poll interval for wireless device data from Kismet. Integration is currently configured to return devices seen as active by any Kismet sensor within the last 10 minutes. + global: True + helpLink: elastic-fleet.html + advanced: True + forcedType: string + api_key: + description: API key for Kismet. + global: True + helpLink: elastic-fleet.html + advanced: True + forcedType: string + sensitive: True + enabled_nodes: + description: Fleet nodes with the Kismet integration enabled. Enter one per line. + global: True + helpLink: elastic-fleet.html + advanced: True + forcedType: "[]string" diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index 275bc6a11..1ade49e44 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -19,7 +19,7 @@ NUM_RUNNING=$(pgrep -cf "/bin/bash /sbin/so-elastic-agent-gen-installers") for i in {1..30} do - ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') + ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys?perPage=100" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r '.item.host_urls[]' | paste -sd ',') if [[ $FLEETHOST ]] && [[ $ENROLLMENTOKEN ]]; then break; else sleep 10; fi done @@ -46,7 +46,7 @@ do done printf "\n### Stripping out unused components" -find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -maxdepth 1 -regex '.*fleet.*\|.*packet.*\|.*apm.*\|.*audit.*\|.*heart.*\|.*cloud.*' -delete +find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -maxdepth 1 -regex '.*fleet.*\|.*packet.*\|.*apm.*\|.*heart.*\|.*cloud.*' -delete printf "\n### Tarring everything up again" for OS in "${OSARCH[@]}" @@ -72,5 +72,5 @@ do printf "\n### $GOOS/$GOARCH Installer Generated...\n" done -printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace" +printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace\n" rm -rf /nsm/elastic-agent-workspace diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade index b1ca8c476..b911f5896 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update new file mode 100644 index 000000000..721525668 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update @@ -0,0 +1,90 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. +{% from 'vars/globals.map.jinja' import GLOBALS %} + +. /usr/sbin/so-common + +# Only run on Managers +if ! is_manager_node; then + printf "Not a Manager Node... Exiting" + exit 0 +fi + +# Function to check if an array contains a value +array_contains () { + local array="$1[@]" + local seeking=$2 + local in=1 + for element in "${!array}"; do + if [[ $element == "$seeking" ]]; then + in=0 + break + fi + done + return $in +} + +# Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes) +LOGSTASHNODES='{{ salt['pillar.get']('logstash:nodes', {}) | tojson }}' + +# Initialize an array for new hosts from Fleet Nodes +declare -a NEW_LIST=() + +# Query for Fleet Nodes & add them to the list (Hostname) +if grep -q "fleet" <<< "$LOGSTASHNODES"; then + readarray -t FLEETNODES < <(jq -r '.fleet | keys_unsorted[]' <<< "$LOGSTASHNODES") + for NODE in "${FLEETNODES[@]}"; do + URL="http://$NODE:8443/artifacts/" + NAME="FleetServer_$NODE" + NEW_LIST+=("$URL=$NAME") + done +fi + +# Create an array for expected hosts and their names +declare -A expected_urls=( + ["http://{{ GLOBALS.url_base }}:8443/artifacts/"]="FleetServer_{{ GLOBALS.hostname }}" + ["https://artifacts.elastic.co/downloads/"]="Elastic Artifacts" +) + +# Merge NEW_LIST into expected_urls +for entry in "${NEW_LIST[@]}"; do + # Extract URL and Name from each entry + IFS='=' read -r URL NAME <<< "$entry" + # Add to expected_urls, automatically handling URL as key and NAME as value + expected_urls["$URL"]="$NAME" +done + +# Fetch the current hosts from the API +current_urls=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/agent_download_sources' | jq -r .items[].host) + +# Convert current hosts to an array +IFS=$'\n' read -rd '' -a current_urls_array <<<"$current_urls" + +# Flag to track if any host was added +any_url_added=0 + +# Check each expected host +for host in "${!expected_urls[@]}"; do + array_contains current_urls_array "$host" || { + echo "$host (${expected_urls[$host]}) is missing. Adding it..." + + # Prepare the JSON payload + JSON_STRING=$( jq -n \ + --arg NAME "${expected_urls[$host]}" \ + --arg URL "$host" \ + '{"name":$NAME,"host":$URL}' ) + + # Create the missing host + curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + + # Flag that an artifact URL was added + any_url_added=1 + } + +done + + +if [[ $any_url_added -eq 0 ]]; then + echo "All expected artifact URLs are present. No updates needed." +fi diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update index 5d5b7e7e0..3da6b3e78 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update index 23a206921..b5d6e1bfe 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. @@ -19,64 +21,104 @@ function update_logstash_outputs() { # Update Logstash Outputs curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_logstash" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq } +function update_kafka_outputs() { + # Make sure SSL configuration is included in policy updates for Kafka output. SSL is configured in so-elastic-fleet-setup + SSL_CONFIG=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_kafka" | jq -r '.item.ssl') -# Get current list of Logstash Outputs -RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash') + JSON_STRING=$(jq -n \ + --arg UPDATEDLIST "$NEW_LIST_JSON" \ + --argjson SSL_CONFIG "$SSL_CONFIG" \ + '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}') + # Update Kafka outputs + curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq +} -# Check to make sure that the server responded with good data - else, bail from script -CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON") -if [ "$CHECKSUM" != "so-manager_logstash" ]; then - printf "Failed to query for current Logstash Outputs..." - exit 1 -fi +{% if GLOBALS.pipeline == "KAFKA" %} + # Get current list of Kafka Outputs + RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_kafka') -# Get the current list of Logstash outputs & hash them -CURRENT_LIST=$(jq -c -r '.item.hosts' <<< "$RAW_JSON") -CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}') + # Check to make sure that the server responded with good data - else, bail from script + CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON") + if [ "$CHECKSUM" != "so-manager_kafka" ]; then + printf "Failed to query for current Kafka Outputs..." + exit 1 + fi -declare -a NEW_LIST=() + # Get the current list of kafka outputs & hash them + CURRENT_LIST=$(jq -c -r '.item.hosts' <<< "$RAW_JSON") + CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}') + + declare -a NEW_LIST=() + + # Query for the current Grid Nodes that are running kafka + KAFKANODES=$(salt-call --out=json pillar.get kafka:nodes | jq '.local') + + # Query for Kafka nodes with Broker role and add hostname to list + while IFS= read -r line; do + NEW_LIST+=("$line") + done < <(jq -r 'to_entries | .[] | select(.value.role | contains("broker")) | .key + ":9092"' <<< $KAFKANODES) + + {# If global pipeline isn't set to KAFKA then assume default of REDIS / logstash #} +{% else %} + # Get current list of Logstash Outputs + RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash') + + # Check to make sure that the server responded with good data - else, bail from script + CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON") + if [ "$CHECKSUM" != "so-manager_logstash" ]; then + printf "Failed to query for current Logstash Outputs..." + exit 1 + fi + + # Get the current list of Logstash outputs & hash them + CURRENT_LIST=$(jq -c -r '.item.hosts' <<< "$RAW_JSON") + CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}') + + declare -a NEW_LIST=() + + {# If we select to not send to manager via SOC, then omit the code that adds manager to NEW_LIST #} + {% if ELASTICFLEETMERGED.enable_manager_output %} + # Create array & add initial elements + if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then + NEW_LIST+=("{{ GLOBALS.url_base }}:5055") + else + NEW_LIST+=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055") + fi + {% endif %} + + # Query for FQDN entries & add them to the list + {% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %} + CUSTOMFQDNLIST=('{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}') + readarray -t -d ' ' CUSTOMFQDN < <(printf '%s' "$CUSTOMFQDNLIST") + for CUSTOMNAME in "${CUSTOMFQDN[@]}" + do + NEW_LIST+=("$CUSTOMNAME:5055") + done + {% endif %} + + # Query for the current Grid Nodes that are running Logstash + LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') + + # Query for Receiver Nodes & add them to the list + if grep -q "receiver" <<< $LOGSTASHNODES; then + readarray -t RECEIVERNODES < <(jq -r ' .receiver | keys_unsorted[]' <<< $LOGSTASHNODES) + for NODE in "${RECEIVERNODES[@]}" + do + NEW_LIST+=("$NODE:5055") + done + fi + + # Query for Fleet Nodes & add them to the list + if grep -q "fleet" <<< $LOGSTASHNODES; then + readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]' <<< $LOGSTASHNODES) + for NODE in "${FLEETNODES[@]}" + do + NEW_LIST+=("$NODE:5055") + done + fi -{# If we select to not send to manager via SOC, then omit the code that adds manager to NEW_LIST #} -{% if ELASTICFLEETMERGED.enable_manager_output %} -# Create array & add initial elements -if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then - NEW_LIST+=("{{ GLOBALS.url_base }}:5055") -else - NEW_LIST+=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055") -fi {% endif %} -# Query for FQDN entries & add them to the list -{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %} -CUSTOMFQDNLIST=('{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}') -readarray -t -d ' ' CUSTOMFQDN < <(printf '%s' "$CUSTOMFQDNLIST") -for CUSTOMNAME in "${CUSTOMFQDN[@]}" -do - NEW_LIST+=("$CUSTOMNAME:5055") -done -{% endif %} - -# Query for the current Grid Nodes that are running Logstash -LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') - -# Query for Receiver Nodes & add them to the list -if grep -q "receiver" <<< $LOGSTASHNODES; then - readarray -t RECEIVERNODES < <(jq -r ' .receiver | keys_unsorted[]' <<< $LOGSTASHNODES) - for NODE in "${RECEIVERNODES[@]}" - do - NEW_LIST+=("$NODE:5055") - done -fi - -# Query for Fleet Nodes & add them to the list -if grep -q "fleet" <<< $LOGSTASHNODES; then - readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]' <<< $LOGSTASHNODES) - for NODE in "${FLEETNODES[@]}" - do - NEW_LIST+=("$NODE:5055") - done -fi - # Sort & hash the new list of Logstash Outputs NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}") NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') @@ -85,9 +127,28 @@ NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then printf "\nHashes match - no update needed.\n" printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + + # Since output can be KAFKA or LOGSTASH, we need to check if the policy set as default matches the value set in GLOBALS.pipeline and update if needed + printf "Checking if the correct output policy is set as default\n" + OUTPUT_DEFAULT=$(jq -r '.item.is_default' <<< $RAW_JSON) + OUTPUT_DEFAULT_MONITORING=$(jq -r '.item.is_default_monitoring' <<< $RAW_JSON) + if [[ "$OUTPUT_DEFAULT" = "false" || "$OUTPUT_DEFAULT_MONITORING" = "false" ]]; then + printf "Default output policy needs to be updated.\n" + {%- if GLOBALS.pipeline == "KAFKA" and 'gmd' in salt['pillar.get']('features', []) %} + update_kafka_outputs + {%- else %} + update_logstash_outputs + {%- endif %} + else + printf "Default output policy is set - no update needed.\n" + fi exit 0 else printf "\nHashes don't match - update needed.\n" printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + {%- if GLOBALS.pipeline == "KAFKA" and 'gmd' in salt['pillar.get']('features', []) %} + update_kafka_outputs + {%- else %} update_logstash_outputs + {%- endif %} fi diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 361469b26..0748557fd 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -77,6 +77,11 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl printf "\n\n" {%- endif %} +printf "\nCreate Kafka Output Config if node is not an Import or Eval install\n" +{% if grains.role not in ['so-import', 'so-eval'] %} +/usr/sbin/so-kafka-fleet-output-policy +{% endif %} + # Add Manager Hostname & URL Base to Fleet Host URLs printf "\nAdd SO-Manager Fleet URL\n" if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index 31c7becca..5f7637cd3 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. diff --git a/salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy b/salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy new file mode 100644 index 000000000..7727f7db1 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% if GLOBALS.role in ['so-manager', 'so-standalone', 'so-managersearch'] %} + +. /usr/sbin/so-common + +# Check to make sure that Kibana API is up & ready +RETURN_CODE=0 +wait_for_web_response "http://localhost:5601/api/fleet/settings" "fleet" 300 "curl -K /opt/so/conf/elasticsearch/curl.config" +RETURN_CODE=$? + +if [[ "$RETURN_CODE" != "0" ]]; then + printf "Kibana API not accessible, can't setup Elastic Fleet output policy for Kafka..." + exit 1 +fi + +output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id) + +if ! echo "$output" | grep -q "so-manager_kafka"; then + KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt) + KAFKAKEY=$(openssl rsa -in /etc/pki/elasticfleet-kafka.key) + KAFKACA=$(openssl x509 -in /etc/pki/tls/certs/intca.crt) + KAFKA_OUTPUT_VERSION="2.6.0" + JSON_STRING=$( jq -n \ + --arg KAFKACRT "$KAFKACRT" \ + --arg KAFKAKEY "$KAFKAKEY" \ + --arg KAFKACA "$KAFKACA" \ + --arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \ + --arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \ + '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 1 }, "topics":[{"topic":"%{[event.module]}-securityonion","when":{"type":"regexp","condition":"event.module:.+"}},{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }' + ) + curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" -o /dev/null + refresh_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id) + + if ! echo "$refresh_output" | grep -q "so-manager_kafka"; then + echo -e "\nFailed to setup Elastic Fleet output policy for Kafka...\n" + exit 1 + elif echo "$refresh_output" | grep -q "so-manager_kafka"; then + echo -e "\nSuccessfully setup Elastic Fleet output policy for Kafka...\n" + fi + +elif echo "$output" | grep -q "so-manager_kafka"; then + echo -e "\nElastic Fleet output policy for Kafka already exists...\n" +fi +{% else %} +echo -e "\nNo update required...\n" +{% endif %} \ No newline at end of file diff --git a/salt/elasticsearch/ca.sls b/salt/elasticsearch/ca.sls index 5485bb676..188450311 100644 --- a/salt/elasticsearch/ca.sls +++ b/salt/elasticsearch/ca.sls @@ -4,7 +4,7 @@ # Elastic License 2.0. {% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} +{% if sls.split('.')[0] in allowed_states or sls in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} # Move our new CA over so Elastic and Logstash can use SSL with the internal CA diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index 23e11a710..27a8a0fd6 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -118,6 +118,19 @@ esingestconf: - user: 930 - group: 939 +# Auto-generate Elasticsearch ingest node pipelines from pillar +{% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %} +es_ingest_conf_{{pipeline}}: + file.managed: + - name: /opt/so/conf/elasticsearch/ingest/{{ pipeline }} + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ config }} + - template: jinja + - onchanges_in: + - file: so-pipelines-reload +{% endfor %} + eslog4jfile: file.managed: - name: /opt/so/conf/elasticsearch/log4j2.properties diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index e35cec326..36d673d70 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,4 +1,6 @@ elasticsearch: + enabled: false + index_clean: true config: action: destructive_requires_name: true @@ -54,7 +56,87 @@ elasticsearch: enabled: true key: /usr/share/elasticsearch/config/elasticsearch.key verification_mode: none - enabled: false + pipelines: + custom001: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom001 + - pipeline: + name: common + custom002: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom002 + - pipeline: + name: common + custom003: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom003 + - pipeline: + name: common + custom004: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom004 + - pipeline: + name: common + custom005: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom005 + - pipeline: + name: common + custom006: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom006 + - pipeline: + name: common + custom007: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom007 + - pipeline: + name: common + custom008: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom008 + - pipeline: + name: common + custom009: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom009 + - pipeline: + name: common + custom010: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom010 + - pipeline: + name: common index_settings: global_overrides: index_template: @@ -70,7 +152,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -88,6 +170,78 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-items: + index_sorting: false + index_template: + composed_of: + - so-items-mappings + index_patterns: + - .items-default-** + priority: 500 + template: + mappings: + date_detection: false + settings: + index: + lifecycle: + name: so-items-logs + rollover_alias: ".items-default" + routing: + allocation: + include: + _tier_preference: "data_content" + mapping: + total_fields: + limit: 10000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: + rollover: + max_size: 50gb + min_age: 0ms + so-lists: + index_sorting: false + index_template: + composed_of: + - so-lists-mappings + index_patterns: + - .lists-default-** + priority: 500 + template: + mappings: + date_detection: false + settings: + index: + lifecycle: + name: so-lists-logs + rollover_alias: ".lists-default" + routing: + allocation: + include: + _tier_preference: "data_content" + mapping: + total_fields: + limit: 10000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: + rollover: + max_size: 50gb + min_age: 0ms so-case: index_sorting: false index_template: @@ -117,6 +271,142 @@ elasticsearch: sort: field: '@timestamp' order: desc + so-detection: + index_sorting: false + index_template: + composed_of: + - detection-mappings + - detection-settings + index_patterns: + - so-detection* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + mapping: + total_fields: + limit: 1500 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + so-logs-soc: + close: 30 + delete: 365 + index_sorting: false + index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - container-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - threat-mappings + - tls-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - common-settings + - common-dynamic-mappings + data_stream: {} + index_patterns: + - logs-soc-so* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + lifecycle: + name: so-soc-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + warm: 7 so-common: close: 30 delete: 365 @@ -214,7 +504,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -326,7 +616,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -431,7 +721,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -544,7 +834,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -652,7 +942,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -715,7 +1005,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -759,7 +1049,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -803,7 +1093,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -847,7 +1137,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -891,7 +1181,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -935,7 +1225,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -979,7 +1269,51 @@ elasticsearch: actions: set_priority: priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 min_age: 30d + so-logs-aws_x_cloudfront_logs: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.cloudfront_logs-*" + template: + settings: + index: + lifecycle: + name: so-logs-aws.cloudfront_logs-logs + number_of_replicas: 0 + composed_of: + - "logs-aws.cloudfront_logs@package" + - "logs-aws.cloudfront_logs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d delete: actions: delete: {} @@ -1023,7 +1357,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1067,7 +1401,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1111,7 +1445,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1155,7 +1489,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1199,7 +1533,95 @@ elasticsearch: actions: set_priority: priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 min_age: 30d + so-logs-aws_x_guardduty: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.guardduty-*" + template: + settings: + index: + lifecycle: + name: so-logs-aws.guardduty-logs + number_of_replicas: 0 + composed_of: + - "logs-aws.guardduty@package" + - "logs-aws.guardduty@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-aws_x_inspector: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.inspector-*" + template: + settings: + index: + lifecycle: + name: so-logs-aws.inspector-logs + number_of_replicas: 0 + composed_of: + - "logs-aws.inspector@package" + - "logs-aws.inspector@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d delete: actions: delete: {} @@ -1243,7 +1665,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1287,7 +1709,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1331,7 +1753,95 @@ elasticsearch: actions: set_priority: priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 min_age: 30d + so-logs-aws_x_securityhub_findings: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.securityhub_findings-*" + template: + settings: + index: + lifecycle: + name: so-logs-aws.securityhub_findings-logs + number_of_replicas: 0 + composed_of: + - "logs-aws.securityhub_findings@package" + - "logs-aws.securityhub_findings@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-aws_x_securityhub_insights: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.securityhub_insights-*" + template: + settings: + index: + lifecycle: + name: so-logs-aws.securityhub_insights-logs + number_of_replicas: 0 + composed_of: + - "logs-aws.securityhub_insights@package" + - "logs-aws.securityhub_insights@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d delete: actions: delete: {} @@ -1375,7 +1885,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1419,7 +1929,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1463,7 +1973,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1507,7 +2017,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1551,7 +2061,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1595,7 +2105,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1639,7 +2149,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1683,7 +2193,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1727,7 +2237,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1771,7 +2281,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1815,7 +2325,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1859,7 +2369,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1903,7 +2413,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1947,7 +2457,51 @@ elasticsearch: actions: set_priority: priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 min_age: 30d + so-logs-cef_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-cef.log-*" + template: + settings: + index: + lifecycle: + name: so-logs-cef.log-logs + number_of_replicas: 0 + composed_of: + - "logs-cef.log@package" + - "logs-cef.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d delete: actions: delete: {} @@ -1991,7 +2545,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2035,7 +2589,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2079,7 +2633,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2123,7 +2677,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2167,7 +2721,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2211,7 +2765,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2255,7 +2809,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2299,7 +2853,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2343,7 +2897,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2387,7 +2941,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2431,7 +2985,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2475,7 +3029,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2519,7 +3073,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2563,7 +3117,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2607,7 +3161,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2651,7 +3205,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2695,7 +3249,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2739,7 +3293,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2783,7 +3337,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2827,7 +3381,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2871,7 +3425,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2915,7 +3469,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2959,7 +3513,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3003,7 +3557,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3047,7 +3601,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3091,7 +3645,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3109,6 +3663,68 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-detections_x_alerts: + index_sorting: false + index_template: + composed_of: + - so-data-streams-mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + - so-logs-mappings + - so-logs-settings + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-detections.alerts-* + priority: 501 + template: + mappings: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + settings: + index: + lifecycle: + name: so-logs-detections.alerts-so + mapping: + total_fields: + limit: 5001 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 1d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-elastic_agent: index_sorting: false index_template: @@ -3153,7 +3769,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3214,7 +3830,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3275,7 +3891,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3333,7 +3949,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3389,7 +4005,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3445,7 +4061,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3498,7 +4114,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3556,7 +4172,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3612,7 +4228,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3668,7 +4284,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3729,7 +4345,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3785,7 +4401,63 @@ elasticsearch: actions: set_priority: priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 min_age: 30d + so-logs-endpoint_x_diagnostic_x_collection: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-endpoint.diagnostic.collection@custom + - logs-endpoint.diagnostic.collection@package + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - .logs-endpoint.diagnostic.collection-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-endpoint.diagnostic.collection-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d delete: actions: delete: {} @@ -3841,7 +4513,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3897,7 +4569,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3953,7 +4625,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4009,7 +4681,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4065,7 +4737,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4121,7 +4793,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4177,7 +4849,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4221,7 +4893,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4265,7 +4937,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4309,7 +4981,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4353,7 +5025,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4397,7 +5069,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4441,7 +5113,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4485,7 +5157,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4529,7 +5201,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4573,7 +5245,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4617,7 +5289,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4661,7 +5333,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4705,7 +5377,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4749,7 +5421,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4793,7 +5465,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4837,7 +5509,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4881,7 +5553,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4925,7 +5597,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4969,7 +5641,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5013,7 +5685,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5057,7 +5729,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5101,7 +5773,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5145,7 +5817,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5189,7 +5861,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5233,7 +5905,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5277,7 +5949,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5321,7 +5993,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5365,7 +6037,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5409,7 +6081,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5453,7 +6125,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5497,7 +6169,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5541,7 +6213,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5585,7 +6257,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5629,7 +6301,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5673,7 +6345,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5717,7 +6389,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5761,7 +6433,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5805,7 +6477,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5849,7 +6521,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5893,7 +6565,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5937,7 +6609,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5981,7 +6653,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6025,7 +6697,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6069,7 +6741,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6113,7 +6785,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6157,7 +6829,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6201,7 +6873,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6245,7 +6917,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6289,7 +6961,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6333,7 +7005,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6377,7 +7049,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6421,7 +7093,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6465,7 +7137,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6509,7 +7181,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6553,7 +7225,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6597,7 +7269,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6641,7 +7313,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6685,7 +7357,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6729,7 +7401,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6773,7 +7445,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6817,7 +7489,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6861,7 +7533,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6905,7 +7577,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6949,7 +7621,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6993,7 +7665,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7037,7 +7709,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7081,7 +7753,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7125,7 +7797,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7203,7 +7875,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7247,7 +7919,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7291,7 +7963,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7335,7 +8007,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7379,7 +8051,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7423,7 +8095,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7467,7 +8139,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7511,7 +8183,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7555,7 +8227,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7599,7 +8271,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7643,7 +8315,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7687,7 +8359,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7731,7 +8403,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7775,7 +8447,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7819,7 +8491,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7863,7 +8535,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7907,7 +8579,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7951,7 +8623,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7995,7 +8667,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8039,7 +8711,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8083,7 +8755,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8128,7 +8800,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8173,7 +8845,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8218,7 +8890,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8263,7 +8935,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8308,7 +8980,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8352,7 +9024,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8396,7 +9068,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8440,7 +9112,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8484,7 +9156,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8528,7 +9200,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8572,7 +9244,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8616,7 +9288,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8660,7 +9332,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8704,7 +9376,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8748,7 +9420,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8792,7 +9464,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8836,7 +9508,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8853,7 +9525,7 @@ elasticsearch: actions: set_priority: priority: 50 - min_age: 30d + min_age: 30d so-logs-ti_otx_x_threat: index_sorting: false index_template: @@ -8880,7 +9552,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8924,7 +9596,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8968,7 +9640,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9012,7 +9684,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9056,7 +9728,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9100,7 +9772,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9144,7 +9816,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9188,7 +9860,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9232,7 +9904,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9276,7 +9948,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9320,7 +9992,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9364,7 +10036,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9408,7 +10080,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9452,7 +10124,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9496,7 +10168,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9540,7 +10212,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9584,7 +10256,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9628,7 +10300,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9672,7 +10344,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9716,7 +10388,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9760,7 +10432,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9804,7 +10476,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9848,7 +10520,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9892,7 +10564,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9936,7 +10608,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9980,7 +10652,54 @@ elasticsearch: actions: set_priority: priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 min_age: 30d + so-kismet: + index_sorting: false + index_template: + composed_of: + - kismet-mappings + - source-mappings + - client-mappings + - device-mappings + - network-mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-kismet-so* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-kismet-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d delete: actions: delete: {} @@ -10090,7 +10809,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10200,7 +10919,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10312,7 +11031,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10423,7 +11142,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10431,7 +11150,118 @@ elasticsearch: hot: actions: rollover: - max_age: 30d + max_age: 1d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-suricata_x_alerts: + index_sorting: false + index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - cloud-mappings + - container-mappings + - data_stream-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - orchestrator-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - registry-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - suricata-mappings + - threat-mappings + - tls-mappings + - tracing-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - vulnerability-mappings + - common-settings + - common-dynamic-mappings + data_stream: {} + index_patterns: + - logs-suricata.alerts-* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + lifecycle: + name: so-suricata.alerts-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 1d max_primary_shard_size: 50gb set_priority: priority: 100 @@ -10535,7 +11365,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10648,7 +11478,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} diff --git a/salt/mysql/disabled.sls b/salt/elasticsearch/download.sls similarity index 67% rename from salt/mysql/disabled.sls rename to salt/elasticsearch/download.sls index 805a755e4..f74c7059a 100644 --- a/salt/mysql/disabled.sls +++ b/salt/elasticsearch/download.sls @@ -1,22 +1,15 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} -include: - - mysql.sostatus - -so-mysql: - docker_container.absent: - - force: True - -so-mysql_so-status.disabled: - file.comment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-mysql$ +so-elasticsearch_image: + docker_image.present: + - name: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ GLOBALS.so_version }} {% else %} diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index b9c66f231..1d35d3505 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -200,9 +200,15 @@ so-elasticsearch-roles-load: - require: - docker_container: so-elasticsearch - file: elasticsearch_sbin_jinja -{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} + +{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} +{% if ELASTICSEARCHMERGED.index_clean %} +{% set ap = "present" %} +{% else %} +{% set ap = "absent" %} +{% endif %} so-elasticsearch-indices-delete: - cron.present: + cron.{{ap}}: - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1 - identifier: so-elasticsearch-indices-delete - user: root @@ -211,7 +217,8 @@ so-elasticsearch-indices-delete: - daymonth: '*' - month: '*' - dayweek: '*' -{% endif %} +{% endif %} + {% endif %} {% else %} diff --git a/salt/elasticsearch/files/ingest-dynamic/common b/salt/elasticsearch/files/ingest-dynamic/common index b2c13f983..836b8d4af 100644 --- a/salt/elasticsearch/files/ingest-dynamic/common +++ b/salt/elasticsearch/files/ingest-dynamic/common @@ -57,10 +57,11 @@ { "convert": { "field": "log.id.uid", "type": "string", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "agent.id", "type": "string", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "event.severity", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, - { "set": { "field": "event.dataset", "ignore_empty_value":true, "copy_from": "event.dataset_temp" }}, + { "set": { "field": "event.dataset", "ignore_empty_value":true, "copy_from": "event.dataset_temp" } }, { "set": { "if": "ctx.event?.dataset != null && !ctx.event.dataset.contains('.')", "field": "event.dataset", "value": "{{event.module}}.{{event.dataset}}" } }, - { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } }, - { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}" }}, + { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } }, + { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}" } }, + { "grok": { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} }, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } } {%- endraw %} {%- if HIGHLANDER %} diff --git a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 index 2ecbc3989..233cd647b 100644 --- a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 +++ b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 @@ -68,7 +68,7 @@ "field": "_security", "ignore_missing": true } - }, + }, { "set": { "ignore_failure": true, "field": "event.module", "value": "elastic_agent" } }, { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } }, { "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } }, @@ -80,9 +80,11 @@ { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } }, { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } }, - { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, + { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, + { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, + { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } ], "on_failure": [ diff --git a/salt/elasticsearch/files/ingest/kismet.ad_hoc b/salt/elasticsearch/files/ingest/kismet.ad_hoc new file mode 100644 index 000000000..adfbd7901 --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.ad_hoc @@ -0,0 +1,10 @@ +{ + "processors": [ + { + "rename": { + "field": "message2.kismet_device_base_macaddr", + "target_field": "network.wireless.bssid" + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.ap b/salt/elasticsearch/files/ingest/kismet.ap new file mode 100644 index 000000000..a864c09e4 --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.ap @@ -0,0 +1,50 @@ +{ + "processors": [ + { + "rename": { + "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_cloaked", + "target_field": "network.wireless.ssid_cloaked", + "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_cloaked != null" + } + }, + { + "rename": { + "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_ssid", + "target_field": "network.wireless.ssid", + "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_ssid != null" + } + }, + { + "set": { + "field": "network.wireless.ssid", + "value": "Hidden", + "if": "ctx?.network?.wireless?.ssid_cloaked != null && ctx?.network?.wireless?.ssid_cloaked == 1" + } + }, + { + "rename": { + "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_dot11e_channel_utilization_perc", + "target_field": "network.wireless.channel_utilization", + "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_dot11e_channel_utilization_perc != null" + } + }, + { + "rename": { + "field": "message2.dot11_device.dot11_device_last_bssid", + "target_field": "network.wireless.bssid" + } + }, + { + "foreach": { + "field": "message2.dot11_device.dot11_device_associated_client_map", + "processor": { + "append": { + "field": "network.wireless.associated_clients", + "value": "{{_ingest._key}}" + } + }, + "if": "ctx?.message2?.dot11_device?.dot11_device_associated_client_map != null" + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.bridged b/salt/elasticsearch/files/ingest/kismet.bridged new file mode 100644 index 000000000..b61635e3a --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.bridged @@ -0,0 +1,16 @@ +{ + "processors": [ + { + "rename": { + "field": "message2.kismet_device_base_macaddr", + "target_field": "client.mac" + } + }, + { + "rename": { + "field": "message2.dot11_device.dot11_device_last_bssid", + "target_field": "network.wireless.bssid" + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.client b/salt/elasticsearch/files/ingest/kismet.client new file mode 100644 index 000000000..6da0a071b --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.client @@ -0,0 +1,29 @@ +{ + "processors": [ + { + "rename": { + "field": "message2.kismet_device_base_macaddr", + "target_field": "client.mac" + } + }, + { + "rename": { + "field": "message2.dot11_device.dot11_device_last_bssid", + "target_field": "network.wireless.last_connected_bssid", + "if": "ctx?.message2?.dot11_device?.dot11_device_last_bssid != null" + } + }, + { + "foreach": { + "field": "message2.dot11_device.dot11_device_client_map", + "processor": { + "append": { + "field": "network.wireless.known_connected_bssid", + "value": "{{_ingest._key}}" + } + }, + "if": "ctx?.message2?.dot11_device?.dot11_device_client_map != null" + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.common b/salt/elasticsearch/files/ingest/kismet.common new file mode 100644 index 000000000..14d439105 --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.common @@ -0,0 +1,159 @@ +{ + "processors": [ + { + "json": { + "field": "message", + "target_field": "message2" + } + }, + { + "date": { + "field": "message2.kismet_device_base_mod_time", + "formats": [ + "epoch_second" + ], + "target_field": "@timestamp" + } + }, + { + "set": { + "field": "event.category", + "value": "network" + } + }, + { + "dissect": { + "field": "message2.kismet_device_base_type", + "pattern": "%{wifi} %{device_type}" + } + }, + { + "lowercase": { + "field": "device_type" + } + }, + { + "set": { + "field": "event.dataset", + "value": "kismet.{{device_type}}" + } + }, + { + "set": { + "field": "event.dataset", + "value": "kismet.wds_ap", + "if": "ctx?.device_type == 'wds ap'" + } + }, + { + "set": { + "field": "event.dataset", + "value": "kismet.ad_hoc", + "if": "ctx?.device_type == 'ad-hoc'" + } + }, + { + "set": { + "field": "event.module", + "value": "kismet" + } + }, + { + "rename": { + "field": "message2.kismet_device_base_packets_tx_total", + "target_field": "source.packets" + } + }, + { + "rename": { + "field": "message2.kismet_device_base_num_alerts", + "target_field": "kismet.alerts.count" + } + }, + { + "rename": { + "field": "message2.kismet_device_base_channel", + "target_field": "network.wireless.channel", + "if": "ctx?.message2?.kismet_device_base_channel != ''" + } + }, + { + "rename": { + "field": "message2.kismet_device_base_frequency", + "target_field": "network.wireless.frequency", + "if": "ctx?.message2?.kismet_device_base_frequency != 0" + } + }, + { + "rename": { + "field": "message2.kismet_device_base_last_time", + "target_field": "kismet.last_seen" + } + }, + { + "date": { + "field": "kismet.last_seen", + "formats": [ + "epoch_second" + ], + "target_field": "kismet.last_seen" + } + }, + { + "rename": { + "field": "message2.kismet_device_base_first_time", + "target_field": "kismet.first_seen" + } + }, + { + "date": { + "field": "kismet.first_seen", + "formats": [ + "epoch_second" + ], + "target_field": "kismet.first_seen" + } + }, + { + "rename": { + "field": "message2.kismet_device_base_seenby", + "target_field": "kismet.seenby" + } + }, + { + "foreach": { + "field": "kismet.seenby", + "processor": { + "pipeline": { + "name": "kismet.seenby" + } + } + } + }, + { + "rename": { + "field": "message2.kismet_device_base_manuf", + "target_field": "device.manufacturer" + } + }, + { + "pipeline": { + "name": "{{event.dataset}}" + } + }, + { + "remove": { + "field": [ + "message2", + "message", + "device_type", + "wifi", + "agent", + "host", + "event.created" + ], + "ignore_failure": true + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.device b/salt/elasticsearch/files/ingest/kismet.device new file mode 100644 index 000000000..49d0c7ad7 --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.device @@ -0,0 +1,9 @@ +{ + "processors": [ + { + "pipeline": { + "name": "kismet.client" + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.seenby b/salt/elasticsearch/files/ingest/kismet.seenby new file mode 100644 index 000000000..d41220d76 --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.seenby @@ -0,0 +1,52 @@ +{ + "processors": [ + { + "rename": { + "field": "_ingest._value.kismet_common_seenby_num_packets", + "target_field": "_ingest._value.packets_seen", + "ignore_missing": true + } + }, + { + "rename": { + "field": "_ingest._value.kismet_common_seenby_uuid", + "target_field": "_ingest._value.serial_number", + "ignore_missing": true + } + }, + { + "rename": { + "field": "_ingest._value.kismet_common_seenby_first_time", + "target_field": "_ingest._value.first_seen", + "ignore_missing": true + } + }, + { + "rename": { + "field": "_ingest._value.kismet_common_seenby_last_time", + "target_field": "_ingest._value.last_seen", + "ignore_missing": true + } + }, + { + "date": { + "field": "_ingest._value.first_seen", + "formats": [ + "epoch_second" + ], + "target_field": "_ingest._value.first_seen", + "ignore_failure": true + } + }, + { + "date": { + "field": "_ingest._value.last_seen", + "formats": [ + "epoch_second" + ], + "target_field": "_ingest._value.last_seen", + "ignore_failure": true + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.wds b/salt/elasticsearch/files/ingest/kismet.wds new file mode 100644 index 000000000..1e426c463 --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.wds @@ -0,0 +1,10 @@ +{ + "processors": [ + { + "rename": { + "field": "message2.kismet_device_base_macaddr", + "target_field": "client.mac" + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.wds_ap b/salt/elasticsearch/files/ingest/kismet.wds_ap new file mode 100644 index 000000000..4d10b211b --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.wds_ap @@ -0,0 +1,22 @@ +{ + "processors": [ + { + "rename": { + "field": "message2.kismet_device_base_commonname", + "target_field": "network.wireless.bssid" + } + }, + { + "foreach": { + "field": "message2.dot11_device.dot11_device_associated_client_map", + "processor": { + "append": { + "field": "network.wireless.associated_clients", + "value": "{{_ingest._key}}" + } + }, + "if": "ctx?.message2?.dot11_device?.dot11_device_associated_client_map != null" + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0 b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0 new file mode 100644 index 000000000..af31e1518 --- /dev/null +++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0 @@ -0,0 +1,389 @@ +{ + "description": "Pipeline for pfSense", + "processors": [ + { + "set": { + "field": "ecs.version", + "value": "8.10.0" + } + }, + { + "set": { + "field": "observer.vendor", + "value": "netgate" + } + }, + { + "set": { + "field": "observer.type", + "value": "firewall" + } + }, + { + "rename": { + "field": "message", + "target_field": "event.original" + } + }, + { + "set": { + "field": "event.kind", + "value": "event" + } + }, + { + "set": { + "field": "event.timezone", + "value": "{{_tmp.tz_offset}}", + "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'" + } + }, + { + "grok": { + "description": "Parse syslog header", + "field": "event.original", + "patterns": [ + "^(%{ECS_SYSLOG_PRI})?%{TIMESTAMP} %{GREEDYDATA:message}" + ], + "pattern_definitions": { + "ECS_SYSLOG_PRI": "<%{NONNEGINT:log.syslog.priority:long}>(\\d )?", + "TIMESTAMP": "(?:%{BSD_TIMESTAMP_FORMAT}|%{SYSLOG_TIMESTAMP_FORMAT})", + "BSD_TIMESTAMP_FORMAT": "%{SYSLOGTIMESTAMP:_tmp.timestamp}(%{SPACE}%{BSD_PROCNAME}|%{SPACE}%{OBSERVER}%{SPACE}%{BSD_PROCNAME})(\\[%{POSINT:process.pid:long}\\])?:", + "BSD_PROCNAME": "(?:\\b%{NAME:process.name}|\\(%{NAME:process.name}\\))", + "NAME": "[[[:alnum:]]_-]+", + "SYSLOG_TIMESTAMP_FORMAT": "%{TIMESTAMP_ISO8601:_tmp.timestamp8601}%{SPACE}%{OBSERVER}%{SPACE}%{PROCESS}%{SPACE}(%{POSINT:process.pid:long}|-) - (-|%{META})", + "TIMESTAMP_ISO8601": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE:event.timezone}?", + "OBSERVER": "(?:%{IP:observer.ip}|%{HOSTNAME:observer.name})", + "PROCESS": "(\\(%{DATA:process.name}\\)|(?:%{UNIXPATH}*/)?%{BASEPATH:process.name})", + "BASEPATH": "[[[:alnum:]]_%!$@:.,+~-]+", + "META": "\\[[^\\]]*\\]" + } + } + }, + { + "date": { + "if": "ctx._tmp.timestamp8601 != null", + "field": "_tmp.timestamp8601", + "target_field": "@timestamp", + "formats": [ + "ISO8601" + ] + } + }, + { + "date": { + "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null", + "field": "_tmp.timestamp", + "target_field": "@timestamp", + "formats": [ + "MMM d HH:mm:ss", + "MMM d HH:mm:ss", + "MMM dd HH:mm:ss" + ], + "timezone": "{{ event.timezone }}" + } + }, + { + "grok": { + "description": "Set Event Provider", + "field": "process.name", + "patterns": [ + "^%{HYPHENATED_WORDS:event.provider}" + ], + "pattern_definitions": { + "HYPHENATED_WORDS": "\\b[A-Za-z0-9_]+(-[A-Za-z_]+)*\\b" + } + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.16.0-firewall", + "if": "ctx.event.provider == 'filterlog'" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.16.0-openvpn", + "if": "ctx.event.provider == 'openvpn'" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.16.0-ipsec", + "if": "ctx.event.provider == 'charon'" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.16.0-dhcp", + "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.16.0-unbound", + "if": "ctx.event.provider == 'unbound'" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.16.0-haproxy", + "if": "ctx.event.provider == 'haproxy'" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.16.0-php-fpm", + "if": "ctx.event.provider == 'php-fpm'" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.16.0-squid", + "if": "ctx.event.provider == 'squid'" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.16.0-suricata", + "if": "ctx.event.provider == 'suricata'" + } + }, + { + "drop": { + "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"suricata\"].contains(ctx.event?.provider)" + } + }, + { + "append": { + "field": "event.category", + "value": "network", + "if": "ctx.network != null" + } + }, + { + "convert": { + "field": "source.address", + "target_field": "source.ip", + "type": "ip", + "ignore_failure": true, + "ignore_missing": true + } + }, + { + "convert": { + "field": "destination.address", + "target_field": "destination.ip", + "type": "ip", + "ignore_failure": true, + "ignore_missing": true + } + }, + { + "set": { + "field": "network.type", + "value": "ipv6", + "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")" + } + }, + { + "set": { + "field": "network.type", + "value": "ipv4", + "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")" + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true + } + }, + { + "geoip": { + "field": "destination.ip", + "target_field": "destination.geo", + "ignore_missing": true + } + }, + { + "geoip": { + "ignore_missing": true, + "database_file": "GeoLite2-ASN.mmdb", + "field": "source.ip", + "target_field": "source.as", + "properties": [ + "asn", + "organization_name" + ] + } + }, + { + "geoip": { + "database_file": "GeoLite2-ASN.mmdb", + "field": "destination.ip", + "target_field": "destination.as", + "properties": [ + "asn", + "organization_name" + ], + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.asn", + "target_field": "source.as.number", + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.organization_name", + "target_field": "source.as.organization.name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "destination.as.asn", + "target_field": "destination.as.number", + "ignore_missing": true + } + }, + { + "rename": { + "field": "destination.as.organization_name", + "target_field": "destination.as.organization.name", + "ignore_missing": true + } + }, + { + "community_id": { + "target_field": "network.community_id", + "ignore_failure": true + } + }, + { + "grok": { + "field": "observer.ingress.interface.name", + "patterns": [ + "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}" + ], + "ignore_missing": true, + "ignore_failure": true + } + }, + { + "set": { + "field": "network.vlan.id", + "copy_from": "observer.ingress.vlan.id", + "ignore_empty_value": true + } + }, + { + "append": { + "field": "related.ip", + "value": "{{destination.ip}}", + "allow_duplicates": false, + "if": "ctx.destination?.ip != null" + } + }, + { + "append": { + "field": "related.ip", + "value": "{{source.ip}}", + "allow_duplicates": false, + "if": "ctx.source?.ip != null" + } + }, + { + "append": { + "field": "related.ip", + "value": "{{source.nat.ip}}", + "allow_duplicates": false, + "if": "ctx.source?.nat?.ip != null" + } + }, + { + "append": { + "field": "related.hosts", + "value": "{{destination.domain}}", + "if": "ctx.destination?.domain != null" + } + }, + { + "append": { + "field": "related.user", + "value": "{{user.name}}", + "if": "ctx.user?.name != null" + } + }, + { + "set": { + "field": "network.direction", + "value": "{{network.direction}}bound", + "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/" + } + }, + { + "remove": { + "field": [ + "_tmp" + ], + "ignore_failure": true + } + }, + { + "script": { + "lang": "painless", + "description": "This script processor iterates over the whole document to remove fields with null values.", + "source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n" + } + }, + { + "remove": { + "field": "event.original", + "if": "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))", + "ignore_failure": true, + "ignore_missing": true + } + }, + { + "pipeline": { + "name": "logs-pfsense.log@custom", + "ignore_missing_pipeline": true + } + } + ], + "on_failure": [ + { + "remove": { + "field": [ + "_tmp" + ], + "ignore_failure": true + } + }, + { + "set": { + "field": "event.kind", + "value": "pipeline_error" + } + }, + { + "append": { + "field": "error.message", + "value": "{{{ _ingest.on_failure_message }}}" + } + } + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "pfsense" + } + } +} diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0-suricata b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0-suricata new file mode 100644 index 000000000..f3a14af44 --- /dev/null +++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0-suricata @@ -0,0 +1,31 @@ +{ + "description": "Pipeline for parsing pfSense Suricata logs.", + "processors": [ + { + "pipeline": { + "name": "suricata.common" + } + } + ], + "on_failure": [ + { + "set": { + "field": "event.kind", + "value": "pipeline_error" + } + }, + { + "append": { + "field": "error.message", + "value": "{{{ _ingest.on_failure_message }}}" + } + } + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "pfsense" + } + } +} diff --git a/salt/elasticsearch/files/ingest/rita.beacon b/salt/elasticsearch/files/ingest/rita.beacons similarity index 100% rename from salt/elasticsearch/files/ingest/rita.beacon rename to salt/elasticsearch/files/ingest/rita.beacons diff --git a/salt/elasticsearch/files/ingest/rita.connection b/salt/elasticsearch/files/ingest/rita.connections similarity index 100% rename from salt/elasticsearch/files/ingest/rita.connection rename to salt/elasticsearch/files/ingest/rita.connections diff --git a/salt/elasticsearch/files/ingest/strelka.file b/salt/elasticsearch/files/ingest/strelka.file index a74a7c622..d9aed3b29 100644 --- a/salt/elasticsearch/files/ingest/strelka.file +++ b/salt/elasticsearch/files/ingest/strelka.file @@ -56,6 +56,7 @@ { "set": { "if": "ctx.exiftool?.Subsystem != null", "field": "host.subsystem", "value": "{{exiftool.Subsystem}}", "ignore_failure": true }}, { "set": { "if": "ctx.scan?.yara?.matches instanceof List", "field": "rule.name", "value": "{{scan.yara.matches.0}}" }}, { "set": { "if": "ctx.rule?.name != null", "field": "event.dataset", "value": "alert", "override": true }}, + { "set": { "if": "ctx.rule?.name != null", "field": "rule.uuid", "value": "{{rule.name}}", "override": true }}, { "rename": { "field": "file.flavors.mime", "target_field": "file.mime_type", "ignore_missing": true }}, { "set": { "if": "ctx.rule?.name != null && ctx.rule?.score == null", "field": "event.severity", "value": 3, "override": true } }, { "convert" : { "if": "ctx.rule?.score != null", "field" : "rule.score","type": "integer"}}, @@ -67,7 +68,8 @@ { "set": { "if": "ctx.scan?.pe?.image_version == '0'", "field": "scan.pe.image_version", "value": "0.0", "override": true } }, { "set": { "field": "observer.name", "value": "{{agent.name}}" }}, { "convert" : { "field" : "scan.exiftool","type": "string", "ignore_missing":true }}, - { "remove": { "field": ["host", "path", "message", "exiftool", "scan.yara.meta"], "ignore_missing": true } }, - { "pipeline": { "name": "common" } } + { "convert" : { "field" : "scan.pe.flags","type": "string", "ignore_missing":true }}, + { "remove": { "field": ["host", "path", "message", "exiftool", "scan.yara.meta"], "ignore_missing": true } }, + { "pipeline": { "name": "common" } } ] } diff --git a/salt/elasticsearch/files/ingest/suricata.alert b/salt/elasticsearch/files/ingest/suricata.alert index 87d5144ed..68e0a5cb3 100644 --- a/salt/elasticsearch/files/ingest/suricata.alert +++ b/salt/elasticsearch/files/ingest/suricata.alert @@ -1,6 +1,7 @@ { "description" : "suricata.alert", "processors" : [ + { "set": { "field": "_index", "value": "logs-suricata.alerts-so" } }, { "set": { "field": "tags","value": "alert" }}, { "rename":{ "field": "message2.alert", "target_field": "rule", "ignore_failure": true } }, { "rename":{ "field": "rule.signature", "target_field": "rule.name", "ignore_failure": true } }, diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common index 6aec40a2b..8143882c7 100644 --- a/salt/elasticsearch/files/ingest/suricata.common +++ b/salt/elasticsearch/files/ingest/suricata.common @@ -13,7 +13,6 @@ { "rename": { "field": "message2.vlan", "target_field": "network.vlan.id", "ignore_failure": true } }, { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_missing": true } }, { "rename": { "field": "message2.xff", "target_field": "xff.ip", "ignore_missing": true } }, - { "lowercase": { "field": "network.transport", "ignore_failure": true } }, { "set": { "field": "event.dataset", "value": "{{ message2.event_type }}" } }, { "set": { "field": "observer.name", "value": "{{agent.name}}" } }, { "set": { "field": "event.ingested", "value": "{{@timestamp}}" } }, diff --git a/salt/elasticsearch/roles/analyst.json b/salt/elasticsearch/roles/analyst.json index 2fd10ab47..90ff95ad4 100644 --- a/salt/elasticsearch/roles/analyst.json +++ b/salt/elasticsearch/roles/analyst.json @@ -27,7 +27,8 @@ "monitor", "read", "read_cross_cluster", - "view_index_metadata" + "view_index_metadata", + "write" ] } ], diff --git a/salt/elasticsearch/roles/limited-analyst.json b/salt/elasticsearch/roles/limited-analyst.json index 9186b732e..6511e5f44 100644 --- a/salt/elasticsearch/roles/limited-analyst.json +++ b/salt/elasticsearch/roles/limited-analyst.json @@ -13,7 +13,8 @@ "monitor", "read", "read_cross_cluster", - "view_index_metadata" + "view_index_metadata", + "write" ] } ], diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index c54e07660..f56ed313e 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -5,6 +5,10 @@ elasticsearch: esheap: description: Specify the memory heap size in (m)egabytes for Elasticsearch. helpLink: elasticsearch.html + index_clean: + description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. + forcedType: bool + helpLink: elasticsearch.html retention: retention_pct: decription: Total percentage of space used by Elasticsearch for multi node clusters @@ -45,6 +49,28 @@ elasticsearch: description: Max number of boolean clauses per query. global: True helpLink: elasticsearch.html + pipelines: + custom001: &pipelines + description: + description: Description of the ingest node pipeline + global: True + advanced: True + helpLink: elasticsearch.html + processors: + description: Processors for the ingest node pipeline + global: True + advanced: True + multiline: True + helpLink: elasticsearch.html + custom002: *pipelines + custom003: *pipelines + custom004: *pipelines + custom005: *pipelines + custom006: *pipelines + custom007: *pipelines + custom008: *pipelines + custom009: *pipelines + custom010: *pipelines index_settings: global_overrides: index_template: @@ -73,12 +99,9 @@ elasticsearch: description: The order to sort by. Must set index_sorting to True. global: True helpLink: elasticsearch.html + policy: phases: hot: - max_age: - description: Maximum age of index. ex. 7d - This determines when the index should be moved out of the hot tier. - global: True - helpLink: elasticsearch.html actions: set_priority: priority: @@ -97,7 +120,9 @@ elasticsearch: helpLink: elasticsearch.html cold: min_age: - description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + regex: ^[0-9]{1,5}d$ + forcedType: string global: True helpLink: elasticsearch.html actions: @@ -108,8 +133,8 @@ elasticsearch: helpLink: elasticsearch.html warm: min_age: - description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. - regex: ^\[0-9\]{1,5}d$ + description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. + regex: ^[0-9]{1,5}d$ forcedType: string global: True actions: @@ -122,6 +147,8 @@ elasticsearch: delete: min_age: description: Minimum age of index. ex. 90d - This determines when the index should be deleted. + regex: ^[0-9]{1,5}d$ + forcedType: string global: True helpLink: elasticsearch.html so-logs: &indexSettings @@ -248,7 +275,9 @@ elasticsearch: helpLink: elasticsearch.html warm: min_age: - description: Minimum age of index. This determines when the index should be moved to the hot tier. + description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. + regex: ^[0-9]{1,5}d$ + forcedType: string global: True advanced: True helpLink: elasticsearch.html @@ -273,7 +302,9 @@ elasticsearch: helpLink: elasticsearch.html cold: min_age: - description: Minimum age of index. This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + regex: ^[0-9]{1,5}d$ + forcedType: string global: True advanced: True helpLink: elasticsearch.html @@ -288,6 +319,8 @@ elasticsearch: delete: min_age: description: Minimum age of index. This determines when the index should be deleted. + regex: ^[0-9]{1,5}d$ + forcedType: string global: True advanced: True helpLink: elasticsearch.html @@ -343,6 +376,7 @@ elasticsearch: so-logs-azure_x_signinlogs: *indexSettings so-logs-azure_x_springcloudlogs: *indexSettings so-logs-barracuda_x_waf: *indexSettings + so-logs-cef_x_log: *indexSettings so-logs-cisco_asa_x_log: *indexSettings so-logs-cisco_ftd_x_log: *indexSettings so-logs-cisco_ios_x_log: *indexSettings @@ -360,6 +394,7 @@ elasticsearch: so-logs-darktrace_x_ai_analyst_alert: *indexSettings so-logs-darktrace_x_model_breach_alert: *indexSettings so-logs-darktrace_x_system_status_alert: *indexSettings + so-logs-detections_x_alerts: *indexSettings so-logs-f5_bigip_x_log: *indexSettings so-logs-fim_x_event: *indexSettings so-logs-fortinet_x_clientendpoint: *indexSettings @@ -486,8 +521,10 @@ elasticsearch: so-endgame: *indexSettings so-idh: *indexSettings so-suricata: *indexSettings + so-suricata_x_alerts: *indexSettings so-import: *indexSettings so-kratos: *indexSettings + so-kismet: *indexSettings so-logstash: *indexSettings so-redis: *indexSettings so-strelka: *indexSettings diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index f5a124a9a..b59c291a4 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -2,11 +2,9 @@ {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %} {% set PILLAR_GLOBAL_OVERRIDES = {} %} -{% if salt['pillar.get']('elasticsearch:index_settings') is defined %} -{% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings') %} -{% if ES_INDEX_PILLAR.global_overrides is defined %} -{% set PILLAR_GLOBAL_OVERRIDES = ES_INDEX_PILLAR.pop('global_overrides') %} -{% endif %} +{% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %} +{% if ES_INDEX_PILLAR.global_overrides is defined %} +{% set PILLAR_GLOBAL_OVERRIDES = ES_INDEX_PILLAR.pop('global_overrides') %} {% endif %} {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %} @@ -19,6 +17,12 @@ {% set ES_INDEX_SETTINGS = {} %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} {% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %} +{# if policy isn't defined in the original index settings, then dont merge policy from the global_overrides #} +{# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #} +{% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} +{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %} +{% endif %} + {% if settings.index_template is defined %} {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} {% do settings.index_template.template.settings.index.pop('sort') %} diff --git a/salt/elasticsearch/templates/component/ecs/device.json b/salt/elasticsearch/templates/component/ecs/device.json new file mode 100644 index 000000000..a281f2c1e --- /dev/null +++ b/salt/elasticsearch/templates/component/ecs/device.json @@ -0,0 +1,36 @@ +{ + "_meta": { + "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-device.html", + "ecs_version": "1.12.2" + }, + "template": { + "mappings": { + "properties": { + "device": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "manufacturer": { + "ignore_above": 1024, + "type": "keyword" + }, + "model": { + "properties": { + "identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/ecs/kismet.json b/salt/elasticsearch/templates/component/ecs/kismet.json new file mode 100644 index 000000000..a03236ab8 --- /dev/null +++ b/salt/elasticsearch/templates/component/ecs/kismet.json @@ -0,0 +1,32 @@ +{ + "_meta": { + "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-base.html", + "ecs_version": "1.12.2" + }, + "template": { + "mappings": { + "properties": { + "kismet": { + "properties": { + "alerts": { + "properties": { + "count": { + "type": "long" + } + } + }, + "first_seen": { + "type": "date" + }, + "last_seen": { + "type": "date" + }, + "seenby": { + "type": "nested" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/ecs/network.json b/salt/elasticsearch/templates/component/ecs/network.json index c2e35efd0..8cc6bdc37 100644 --- a/salt/elasticsearch/templates/component/ecs/network.json +++ b/salt/elasticsearch/templates/component/ecs/network.json @@ -77,6 +77,43 @@ "type": "keyword" } } + }, + "wireless": { + "properties": { + "associated_clients": { + "ignore_above": 1024, + "type": "keyword" + }, + "bssid": { + "ignore_above": 1024, + "type": "keyword" + }, + "channel": { + "ignore_above": 1024, + "type": "keyword" + }, + "channel_utilization": { + "type": "float" + }, + "frequency": { + "type": "double" + }, + "ssid": { + "ignore_above": 1024, + "type": "keyword" + }, + "ssid_cloaked": { + "type": "integer" + }, + "known_connected_bssid": { + "ignore_above": 1024, + "type": "keyword" + }, + "last_connected_bssid": { + "ignore_above": 1024, + "type": "keyword" + } + } } } } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json index 2390705f3..7bcc34de9 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json @@ -1,382 +1,383 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-elastic_agent-1.13.1", - "mapping": { - "total_fields": { - "limit": "10000" +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "logs" + }, + "codec": "best_compression", + "default_pipeline": "logs-elastic_agent-1.13.1", + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "query": { + "default_field": [ + "cloud.account.id", + "cloud.availability_zone", + "cloud.instance.id", + "cloud.instance.name", + "cloud.machine.type", + "cloud.provider", + "cloud.region", + "cloud.project.id", + "cloud.image.id", + "container.id", + "container.image.name", + "container.name", + "host.architecture", + "host.hostname", + "host.id", + "host.mac", + "host.name", + "host.os.family", + "host.os.kernel", + "host.os.name", + "host.os.platform", + "host.os.version", + "host.os.build", + "host.os.codename", + "host.type", + "ecs.version", + "agent.build.original", + "agent.ephemeral_id", + "agent.id", + "agent.name", + "agent.type", + "agent.version", + "log.level", + "message", + "elastic_agent.id", + "elastic_agent.process", + "elastic_agent.version", + "component.id", + "component.type", + "component.binary", + "component.state", + "component.old_state", + "unit.id", + "unit.type", + "unit.state", + "unit.old_state" + ] + } + } + }, + "mappings": { + "dynamic": false, + "dynamic_templates": [ + { + "container.labels": { + "path_match": "container.labels.*", + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "container": { + "properties": { + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "ecs.version", - "agent.build.original", - "agent.ephemeral_id", - "agent.id", - "agent.name", - "agent.type", - "agent.version", - "log.level", - "message", - "elastic_agent.id", - "elastic_agent.process", - "elastic_agent.version", - "component.id", - "component.type", - "component.binary", - "component.state", - "component.old_state", - "unit.id", - "unit.type", - "unit.state", - "unit.old_state" - ] } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" } - }, - "mappings": { - "dynamic": false, - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" + } + }, + "agent": { + "properties": { + "build": { + "properties": { + "original": { + "ignore_above": 1024, + "type": "keyword" } } - ], - "properties": { - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "log": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "elastic_agent": { + "properties": { + "process": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "snapshot": { + "type": "boolean" + } + } + }, + "message": { + "type": "text" + }, + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" } - }, - "agent": { - "properties": { - "build": { - "properties": { - "original": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } + } + }, + "instance": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - } + } + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" } - }, - "elastic_agent": { - "properties": { - "process": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "snapshot": { - "type": "boolean" - } + } + }, + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" } - }, - "message": { - "type": "text" - }, - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "component": { - "properties": { - "binary": { - "ignore_above": 1024, - "type": "keyword" - }, - "old_state": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "wildcard" - }, - "state": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "unit": { - "properties": { - "old_state": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "wildcard" - }, - "state": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event": { - "properties": { - "dataset": { - "type": "constant_keyword" - } + } + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + }, + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" } } } } }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true + "component": { + "properties": { + "binary": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "wildcard" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "unit": { + "properties": { + "old_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "wildcard" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "@timestamp": { + "type": "date" + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + }, + "type": { + "type": "constant_keyword" + }, + "dataset": { + "type": "constant_keyword" + } + } + }, + "host": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "codename": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword", + "fields": { + "text": { + "type": "text" + } + } + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "containerized": { + "type": "boolean" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "architecture": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "dataset": { + "type": "constant_keyword" + } + } } } + } + }, + "_meta": { + "package": { + "name": "elastic_agent" + }, + "managed_by": "fleet", + "managed": true + } +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-endpoint.diagnostic.collection@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-endpoint.diagnostic.collection@custom.json new file mode 100644 index 000000000..1bf9313a9 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-endpoint.diagnostic.collection@custom.json @@ -0,0 +1,12 @@ +{ + "template": { + "settings": {} + }, + "_meta": { + "package": { + "name": "endpoint" + }, + "managed_by": "fleet", + "managed": true + } +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-endpoint.diagnostic.collection@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-endpoint.diagnostic.collection@package.json new file mode 100644 index 000000000..bf60f2543 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-endpoint.diagnostic.collection@package.json @@ -0,0 +1,132 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "logs-endpoint.collection-diagnostic" + }, + "codec": "best_compression", + "default_pipeline": "logs-endpoint.diagnostic.collection-8.10.2", + "mapping": { + "total_fields": { + "limit": "10000" + }, + "ignore_malformed": "true" + }, + "query": { + "default_field": [ + "ecs.version", + "event.action", + "event.category", + "event.code", + "event.dataset", + "event.hash", + "event.id", + "event.kind", + "event.module", + "event.outcome", + "event.provider", + "event.type" + ] + } + } + }, + "mappings": { + "dynamic": false, + "properties": { + "@timestamp": { + "ignore_malformed": false, + "type": "date" + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + }, + "type": { + "type": "constant_keyword" + }, + "dataset": { + "type": "constant_keyword" + } + } + }, + "event": { + "properties": { + "severity": { + "type": "long" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "sequence": { + "type": "long" + }, + "ingested": { + "type": "date" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + }, + "_meta": { + "package": { + "name": "endpoint" + }, + "managed_by": "fleet", + "managed": true + } +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json new file mode 100644 index 000000000..0123fb956 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json @@ -0,0 +1,22 @@ +{ + "template": { + "mappings": { + "properties": { + "error": { + "properties": { + "message": { + "type": "match_only_text" + } + } + } + } + } + }, + "_meta": { + "package": { + "name": "system" + }, + "managed_by": "fleet", + "managed": true + } +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-items-mappings.json b/salt/elasticsearch/templates/component/elastic-agent/so-items-mappings.json new file mode 100644 index 000000000..85e6c1984 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/so-items-mappings.json @@ -0,0 +1,112 @@ +{ + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "binary": { + "type": "binary" + }, + "boolean": { + "type": "boolean" + }, + "byte": { + "type": "byte" + }, + "created_at": { + "type": "date" + }, + "created_by": { + "type": "keyword" + }, + "date": { + "type": "date" + }, + "date_nanos": { + "type": "date_nanos" + }, + "date_range": { + "type": "date_range" + }, + "deserializer": { + "type": "keyword" + }, + "double": { + "type": "double" + }, + "double_range": { + "type": "double_range" + }, + "float": { + "type": "float" + }, + "float_range": { + "type": "float_range" + }, + "geo_point": { + "type": "geo_point" + }, + "geo_shape": { + "type": "geo_shape" + }, + "half_float": { + "type": "half_float" + }, + "integer": { + "type": "integer" + }, + "integer_range": { + "type": "integer_range" + }, + "ip": { + "type": "ip" + }, + "ip_range": { + "type": "ip_range" + }, + "keyword": { + "type": "keyword" + }, + "list_id": { + "type": "keyword" + }, + "long": { + "type": "long" + }, + "long_range": { + "type": "long_range" + }, + "meta": { + "type": "object", + "enabled": false + }, + "serializer": { + "type": "keyword" + }, + "shape": { + "type": "shape" + }, + "short": { + "type": "short" + }, + "text": { + "type": "text" + }, + "tie_breaker_id": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "type": "keyword" + } + } + }, + "aliases": {} + }, + "version": 2, + "_meta": { + "managed": true, + "description": "default mappings for the .items index template installed by Kibana/Security" + } +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-lists-mappings.json b/salt/elasticsearch/templates/component/elastic-agent/so-lists-mappings.json new file mode 100644 index 000000000..b2b5fda23 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/so-lists-mappings.json @@ -0,0 +1,55 @@ +{ + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "created_at": { + "type": "date" + }, + "created_by": { + "type": "keyword" + }, + "description": { + "type": "keyword" + }, + "deserializer": { + "type": "keyword" + }, + "immutable": { + "type": "boolean" + }, + "meta": { + "type": "object", + "enabled": false + }, + "name": { + "type": "keyword" + }, + "serializer": { + "type": "keyword" + }, + "tie_breaker_id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "aliases": {} + }, + "version": 2, + "_meta": { + "managed": true, + "description": "default mappings for the .lists index template installed by Kibana/Security" + } +} diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json new file mode 100644 index 000000000..5e51b872b --- /dev/null +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -0,0 +1,154 @@ +{ + "template": { + "mappings": { + "properties": { + "so_audit_doc_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "@timestamp": { + "type": "date" + }, + "so_kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_operation": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_detection": { + "properties": { + "publicId": { + "ignore_above": 1024, + "type": "keyword" + }, + "title": { + "ignore_above": 1024, + "type": "keyword" + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + }, + "author": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "type": "text" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "ignore_above": 1024, + "type": "keyword" + }, + "content": { + "type": "text" + }, + "isEnabled": { + "type": "boolean" + }, + "isReporting": { + "type": "boolean" + }, + "isCommunity": { + "type": "boolean" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, + "engine": { + "ignore_above": 1024, + "type": "keyword" + }, + "language": { + "ignore_above": 1024, + "type": "keyword" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "overrides": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "isEnabled": { + "type": "boolean" + }, + "createdAt": { + "type": "date" + }, + "updatedAt": { + "type": "date" + }, + "regex": { + "type": "text" + }, + "value": { + "type": "text" + }, + "thresholdType": { + "ignore_above": 1024, + "type": "keyword" + }, + "track": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "text" + }, + "count": { + "type": "long" + }, + "seconds": { + "type": "long" + }, + "customFilter": { + "type": "text" + } + } + } + } + }, + "so_detectioncomment": { + "properties": { + "createTime": { + "type": "date" + }, + "detectionId": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "type": "text" + }, + "userId": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + }, + "_meta": { + "ecs_version": "1.12.2" + } +} diff --git a/salt/elasticsearch/templates/component/so/detection-settings.json b/salt/elasticsearch/templates/component/so/detection-settings.json new file mode 100644 index 000000000..7b0947a4c --- /dev/null +++ b/salt/elasticsearch/templates/component/so/detection-settings.json @@ -0,0 +1,7 @@ +{ + "template": {}, + "version": 1, + "_meta": { + "description": "default settings for common Security Onion Detections indices" + } +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/so/so-scan-mappings.json b/salt/elasticsearch/templates/component/so/so-scan-mappings.json index 008a6ab10..2d11acba0 100644 --- a/salt/elasticsearch/templates/component/so/so-scan-mappings.json +++ b/salt/elasticsearch/templates/component/so/so-scan-mappings.json @@ -14,16 +14,19 @@ }, "pe": { "properties": { - "sections": { + "flags": { + "type": "text" + }, + "image_version": { + "type": "float" + }, + "sections": { "properties": { "entropy": { "type": "float" } } - }, - "image_version": { - "type": "float" - } + } } }, "elf": { diff --git a/salt/elasticsearch/tools/sbin/so-index-list b/salt/elasticsearch/tools/sbin/so-index-list index 1e4595b35..572e55cba 100755 --- a/salt/elasticsearch/tools/sbin/so-index-list +++ b/salt/elasticsearch/tools/sbin/so-index-list @@ -5,6 +5,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +. /usr/sbin/so-common - -curl -K /opt/so/conf/elasticsearch/curl.config-X GET -k -L "https://localhost:9200/_cat/indices?v&s=index" +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L "https://localhost:9200/_cat/indices?pretty&v&s=index" diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total index 3faa2a7a9..b73967c26 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total @@ -40,7 +40,7 @@ fi # Iterate through the output of _cat/allocation for each node in the cluster to determine the total available space {% if GLOBALS.role == 'so-manager' %} -for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v {{ GLOBALS.manager }} | awk '{print $5}'); do +for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $5}'); do {% else %} for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $5}'); do {% endif %} diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used index 5d8a60e22..49e634853 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used @@ -13,7 +13,7 @@ TOTAL_USED_SPACE=0 # Iterate through the output of _cat/allocation for each node in the cluster to determine the total used space {% if GLOBALS.role == 'so-manager' %} # Get total disk space - disk.total -for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v {{ GLOBALS.manager }} | awk '{print $3}'); do +for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $3}'); do {% else %} # Get disk space taken up by indices - disk.indices for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $2}'); do diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete index 07feb36bd..a00437a25 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete @@ -27,6 +27,7 @@ overlimit() { # 2. Check if the maximum number of iterations - MAX_ITERATIONS - has been exceeded. If so, exit. # Closed indices will be deleted first. If we are able to bring disk space under LOG_SIZE_LIMIT, or the number of iterations has exceeded the maximum allowed number of iterations, we will break out of the loop. + while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do # If we can't query Elasticsearch, then immediately return false. @@ -34,28 +35,36 @@ while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do [ $? -eq 1 ] && echo "$(date) - Could not query Elasticsearch." >> ${LOG} && exit # We iterate through the closed and open indices - CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -vE "playbook|so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3) - OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -vE "playbook|so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3) + CLOSED_SO_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -E "(^logstash-.*|^so-.*)" | grep -vE "so-case|so-detection" | sort -t- -k3) + CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -E "^.ds-logs-.*" | grep -v "suricata" | sort -t- -k4) + OPEN_SO_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -E "(^logstash-.*|^so-.*)" | grep -vE "so-case|so-detection" | sort -t- -k3) + OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -E "^.ds-logs-.*" | grep -v "suricata" | sort -t- -k4) - for INDEX in ${CLOSED_INDICES} ${OPEN_INDICES}; do - # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream - # To do so, we need to identify to which data stream this index is associated - # We extract the data stream name using the pattern below - DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+" - DATASTREAM=$(echo "${INDEX}" | grep -oE "$DATASTREAM_PATTERN") - # We look up the data stream, and determine the write index. If there is only one backing index, we delete the entire data stream - BACKING_INDICES=$(/usr/sbin/so-elasticsearch-query _data_stream/${DATASTREAM} | jq -r '.data_streams[0].indices | length') - if [ "$BACKING_INDICES" -gt 1 ]; then - CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name) - # We make sure we are not trying to delete a write index - if [ "${INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then - # This should not be a write index, so we should be allowed to delete it - printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG} - /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 - fi + for INDEX in ${CLOSED_SO_INDICES} ${OPEN_SO_INDICES} ${CLOSED_INDICES} ${OPEN_INDICES}; do + # Check if index is an older index. If it is an older index, delete it before moving on to newer indices. + if [[ "$INDEX" =~ "^logstash-.*|so-.*" ]]; then + printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG} + /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 else - printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - There is only one backing index (${INDEX}). Deleting ${DATASTREAM} data stream...\n" >> ${LOG} + # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream + # To do so, we need to identify to which data stream this index is associated + # We extract the data stream name using the pattern below + DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+" + DATASTREAM=$(echo "${INDEX}" | grep -oE "$DATASTREAM_PATTERN") + # We look up the data stream, and determine the write index. If there is only one backing index, we delete the entire data stream + BACKING_INDICES=$(/usr/sbin/so-elasticsearch-query _data_stream/${DATASTREAM} | jq -r '.data_streams[0].indices | length') + if [ "$BACKING_INDICES" -gt 1 ]; then + CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name) + # We make sure we are not trying to delete a write index + if [ "${INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then + # This should not be a write index, so we should be allowed to delete it + printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG} + /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 + fi + else + printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - There is only one backing index (${INDEX}). Deleting ${DATASTREAM} data stream...\n" >> ${LOG} /usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM -XDELETE >> ${LOG} 2>&1 + fi fi if ! overlimit ; then exit diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 8f45d6c36..080348522 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -133,7 +133,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then for i in $pattern; do TEMPLATE=${i::-14} COMPONENT_PATTERN=${TEMPLATE:3} - MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -v osquery) + MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery") if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" ]]; then load_failures=$((load_failures+1)) echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures" diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index 0ba2389e9..02a1b7cac 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -9,11 +9,9 @@ 'so-influxdb', 'so-kibana', 'so-kratos', - 'so-mysql', 'so-nginx', 'so-redis', 'so-soc', - 'so-soctopus', 'so-strelka-coordinator', 'so-strelka-gatekeeper', 'so-strelka-frontend', @@ -29,14 +27,13 @@ 'so-elastic-fleet', 'so-elastic-fleet-package-registry', 'so-influxdb', + 'so-kafka', 'so-kibana', 'so-kratos', 'so-logstash', - 'so-mysql', 'so-nginx', 'so-redis', 'so-soc', - 'so-soctopus', 'so-strelka-coordinator', 'so-strelka-gatekeeper', 'so-strelka-frontend', @@ -84,6 +81,7 @@ {% set NODE_CONTAINERS = [ 'so-logstash', 'so-redis', + 'so-kafka' ] %} {% elif GLOBALS.role == 'so-idh' %} @@ -95,6 +93,7 @@ {% set NODE_CONTAINERS = [ 'so-elastic-fleet', 'so-logstash', + 'so-nginx-fleet-node' ] %} {% elif GLOBALS.role == 'so-sensor' %} diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 75df49b25..fc5368e12 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -90,27 +90,27 @@ firewall: tcp: - 8086 udp: [] + kafka_controller: + tcp: + - 9093 + udp: [] + kafka_data: + tcp: + - 9092 + udp: [] kibana: tcp: - 5601 udp: [] localrules: - tcp: - - 7788 - udp: [] - mysql: tcp: - - 3306 + - 7788 udp: [] nginx: tcp: - 80 - 443 udp: [] - playbook: - tcp: - - 3000 - udp: [] redis: tcp: - 6379 @@ -178,8 +178,6 @@ firewall: hostgroups: eval: portgroups: - - playbook - - mysql - kibana - redis - influxdb @@ -363,8 +361,6 @@ firewall: hostgroups: manager: portgroups: - - playbook - - mysql - kibana - redis - influxdb @@ -559,8 +555,6 @@ firewall: hostgroups: managersearch: portgroups: - - playbook - - mysql - kibana - redis - influxdb @@ -756,8 +750,6 @@ firewall: - all standalone: portgroups: - - playbook - - mysql - kibana - redis - influxdb @@ -769,7 +761,6 @@ firewall: - beats_5044 - beats_5644 - beats_5056 - - redis - elasticsearch_node - elastic_agent_control - elastic_agent_data @@ -1283,31 +1274,51 @@ firewall: chain: DOCKER-USER: hostgroups: + desktop: + portgroups: + - elastic_agent_data fleet: portgroups: - - beats_5056 + - elastic_agent_data + idh: + portgroups: + - elastic_agent_data sensor: portgroups: - - beats_5044 - - beats_5644 - elastic_agent_data searchnode: portgroups: - redis - - beats_5644 + - elastic_agent_data + standalone: + portgroups: + - redis + - elastic_agent_data + manager: + portgroups: + - elastic_agent_data + managersearch: + portgroups: + - redis + - elastic_agent_data self: portgroups: - redis - - beats_5644 + - elastic_agent_data beats_endpoint: portgroups: - beats_5044 beats_endpoint_ssl: portgroups: - beats_5644 + elastic_agent_endpoint: + portgroups: + - elastic_agent_data endgame: portgroups: - endgame + receiver: + portgroups: [] customhostgroup0: portgroups: [] customhostgroup1: diff --git a/salt/firewall/map.jinja b/salt/firewall/map.jinja index 74b3a66be..fe04d7ad3 100644 --- a/salt/firewall/map.jinja +++ b/salt/firewall/map.jinja @@ -18,4 +18,28 @@ {% endfor %} {% endif %} +{# Only add Kafka firewall items when Kafka enabled #} +{% set role = GLOBALS.role.split('-')[1] %} + +{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone'] %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[role].portgroups.append('kafka_controller') %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %} +{% endif %} + +{% if GLOBALS.pipeline == 'KAFKA' and role == 'receiver' %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.self.portgroups.append('kafka_controller') %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.standalone.portgroups.append('kafka_controller') %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.manager.portgroups.append('kafka_controller') %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.managersearch.portgroups.append('kafka_controller') %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %} +{% endif %} + +{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone', 'receiver'] %} +{% for r in ['manager', 'managersearch', 'standalone', 'receiver', 'fleet', 'idh', 'sensor', 'searchnode','heavynode', 'elastic_agent_endpoint', 'desktop'] %} +{% if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %} +{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('kafka_data') %} +{% endif %} +{% endfor %} +{% endif %} + {% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %} diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 7a2e3b035..91aec0b0d 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -7,6 +7,7 @@ firewall: multiline: True regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ regexFailureMessage: You must enter a valid IP address or CIDR. + duplicates: True anywhere: &hostgroupsettingsadv description: List of IP or CIDR blocks to allow access to this hostgroup. forcedType: "[]string" @@ -15,6 +16,7 @@ firewall: advanced: True regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ regexFailureMessage: You must enter a valid IP address or CIDR. + duplicates: True beats_endpoint: *hostgroupsettings beats_endpoint_ssl: *hostgroupsettings dockernet: &ROhostgroupsettingsadv @@ -53,6 +55,7 @@ firewall: multiline: True regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ regexFailureMessage: You must enter a valid IP address or CIDR. + duplicates: True customhostgroup1: *customhostgroupsettings customhostgroup2: *customhostgroupsettings customhostgroup3: *customhostgroupsettings @@ -70,12 +73,14 @@ firewall: helpLink: firewall.html advanced: True multiline: True + duplicates: True udp: &udpsettings description: List of UDP ports for this port group. forcedType: "[]string" helpLink: firewall.html advanced: True multiline: True + duplicates: True agrules: tcp: *tcpsettings udp: *udpsettings @@ -115,21 +120,18 @@ firewall: influxdb: tcp: *tcpsettings udp: *udpsettings + kafka: + tcp: *tcpsettings + udp: *udpsettings kibana: tcp: *tcpsettings udp: *udpsettings localrules: tcp: *tcpsettings udp: *udpsettings - mysql: - tcp: *tcpsettings - udp: *udpsettings nginx: tcp: *tcpsettings udp: *udpsettings - playbook: - tcp: *tcpsettings - udp: *udpsettings redis: tcp: *tcpsettings udp: *udpsettings @@ -193,6 +195,7 @@ firewall: multiline: True forcedType: "[]string" helpLink: firewall.html + duplicates: True sensor: portgroups: *portgroupsdocker searchnode: @@ -246,6 +249,7 @@ firewall: multiline: True forcedType: "[]string" helpLink: firewall.html + duplicates: True dockernet: portgroups: *portgroupshost localhost: @@ -938,7 +942,6 @@ firewall: portgroups: *portgroupshost customhostgroup9: portgroups: *portgroupshost - idh: chain: DOCKER-USER: diff --git a/salt/global/defaults.yaml b/salt/global/defaults.yaml new file mode 100644 index 000000000..5daa942c8 --- /dev/null +++ b/salt/global/defaults.yaml @@ -0,0 +1,3 @@ +global: + pcapengine: STENO + pipeline: REDIS \ No newline at end of file diff --git a/salt/global/map.jinja b/salt/global/map.jinja new file mode 100644 index 000000000..54abb8c79 --- /dev/null +++ b/salt/global/map.jinja @@ -0,0 +1,2 @@ +{% import_yaml 'global/defaults.yaml' as GLOBALDEFAULTS %} +{% set GLOBALMERGED = salt['pillar.get']('global', GLOBALDEFAULTS.global, merge=True) %} diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index 14d637d50..5a349a3c3 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -10,10 +10,15 @@ global: regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ regexFailureMessage: You must enter a valid IP address or CIDR. mdengine: - description: What engine to use for meta data generation. Options are ZEEK and SURICATA. + description: Which engine to use for meta data generation. Options are ZEEK and SURICATA. regex: ^(ZEEK|SURICATA)$ regexFailureMessage: You must enter either ZEEK or SURICATA. global: True + pcapengine: + description: Which engine to use for generating pcap. Options are STENO, SURICATA or TRANSITION. + regex: ^(STENO|SURICATA|TRANSITION)$ + regexFailureMessage: You must enter either STENO, SURICATA or TRANSITION. + global: True ids: description: Which IDS engine to use. Currently only Suricata is supported. global: True @@ -23,7 +28,7 @@ global: description: Used for handling of authentication cookies. global: True airgap: - description: Sets airgap mode. + description: Airgapped systems do not have network connectivity to the internet. This setting represents how this grid was configured during initial setup. While it is technically possible to manually switch systems between airgap and non-airgap, there are some nuances and additional steps involved. For that reason this setting is marked read-only. Contact your support representative for guidance if there is a need to change this setting. global: True readonly: True imagerepo: @@ -31,9 +36,10 @@ global: global: True advanced: True pipeline: - description: Sets which pipeline technology for events to use. Currently only Redis is supported. + description: Sets which pipeline technology for events to use. Currently only Redis is fully supported. Kafka is experimental and requires a Security Onion Pro license. + regex: ^(REDIS|KAFKA)$ + regexFailureMessage: You must enter either REDIS or KAFKA. global: True - readonly: True advanced: True repo_host: description: Specify the host where operating system packages will be served from. diff --git a/salt/idh/openssh/config.sls b/salt/idh/openssh/config.sls index d358bb5be..5e2acd8d2 100644 --- a/salt/idh/openssh/config.sls +++ b/salt/idh/openssh/config.sls @@ -11,6 +11,8 @@ idh_sshd_selinux: - sel_type: ssh_port_t - prereq: - file: openssh_config + - require: + - pkg: python_selinux_mgmt_tools {% endif %} openssh_config: diff --git a/salt/idh/openssh/init.sls b/salt/idh/openssh/init.sls index ba0a8ab04..79d082502 100644 --- a/salt/idh/openssh/init.sls +++ b/salt/idh/openssh/init.sls @@ -15,3 +15,9 @@ openssh: - enable: False - name: {{ openssh_map.service }} {% endif %} + +{% if grains.os_family == 'RedHat' %} +python_selinux_mgmt_tools: + pkg.installed: + - name: policycoreutils-python-utils +{% endif %} diff --git a/salt/idstools/config.sls b/salt/idstools/config.sls index e162d1139..6d4b1036e 100644 --- a/salt/idstools/config.sls +++ b/salt/idstools/config.sls @@ -33,6 +33,19 @@ idstools_sbin_jinja: - file_mode: 755 - template: jinja +suricatacustomdirsfile: + file.directory: + - name: /nsm/rules/detect-suricata/custom_file + - user: 939 + - group: 939 + - makedirs: True + +suricatacustomdirsurl: + file.directory: + - name: /nsm/rules/detect-suricata/custom_temp + - user: 939 + - group: 939 + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls index decc5a5b2..5e4c4c066 100644 --- a/salt/idstools/enabled.sls +++ b/salt/idstools/enabled.sls @@ -39,7 +39,7 @@ so-idstools: {% endif %} - binds: - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro - - /opt/so/rules/nids:/opt/so/rules/nids:rw + - /opt/so/rules/nids/suri:/opt/so/rules/nids/suri:rw - /nsm/rules/:/nsm/rules/:rw {% if DOCKER.containers['so-idstools'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-idstools'].custom_bind_mounts %} diff --git a/salt/idstools/etc/rulecat.conf b/salt/idstools/etc/rulecat.conf index d6f3d93d8..db78cec29 100644 --- a/salt/idstools/etc/rulecat.conf +++ b/salt/idstools/etc/rulecat.conf @@ -1,17 +1,23 @@ {%- from 'vars/globals.map.jinja' import GLOBALS -%} -{%- from 'idstools/map.jinja' import IDSTOOLSMERGED -%} ---merged=/opt/so/rules/nids/all.rules ---local=/opt/so/rules/nids/local.rules +{%- from 'soc/merged.map.jinja' import SOCMERGED -%} +--suricata-version=6.0 +--merged=/opt/so/rules/nids/suri/all.rules +--output=/nsm/rules/detect-suricata/custom_temp +--local=/opt/so/rules/nids/suri/local.rules {%- if GLOBALS.md_engine == "SURICATA" %} ---local=/opt/so/rules/nids/extraction.rules ---local=/opt/so/rules/nids/filters.rules +--local=/opt/so/rules/nids/suri/extraction.rules +--local=/opt/so/rules/nids/suri/filters.rules {%- endif %} --url=http://{{ GLOBALS.manager }}:7788/suricata/emerging-all.rules --disable=/opt/so/idstools/etc/disable.conf --enable=/opt/so/idstools/etc/enable.conf --modify=/opt/so/idstools/etc/modify.conf -{%- if IDSTOOLSMERGED.config.urls | length > 0 %} -{%- for URL in IDSTOOLSMERGED.config.urls %} ---url={{ URL }} -{%- endfor %} -{%- endif %} +{%- if SOCMERGED.config.server.modules.suricataengine.customRulesets %} + {%- for ruleset in SOCMERGED.config.server.modules.suricataengine.customRulesets %} + {%- if 'url' in ruleset %} +--url={{ ruleset.url }} + {%- elif 'file' in ruleset %} +--local={{ ruleset.file }} + {%- endif %} + {%- endfor %} +{%- endif %} \ No newline at end of file diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml index 634f68803..993abfd51 100644 --- a/salt/idstools/soc_idstools.yaml +++ b/salt/idstools/soc_idstools.yaml @@ -6,45 +6,56 @@ idstools: description: Enter your registration code or oinkcode for paid NIDS rulesets. title: Registration Code global: True + forcedType: string helpLink: rules.html ruleset: - description: Defines the ruleset you want to run. Options are ETOPEN or ETPRO. + description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. Once you have changed the ruleset here, you will need to wait for the rule update to take place (every 24 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Suricata --> Full Update. WARNING! Changing the ruleset will remove all existing non-overlapping Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.' global: True regex: ETPRO\b|ETOPEN\b helpLink: rules.html urls: - description: This is a list of additional rule download locations. + description: This is a list of additional rule download locations. This feature is currently disabled. global: True + multiline: True + forcedType: "[]string" + readonly: True helpLink: rules.html sids: disabled: - description: Contains the list of NIDS rules manually disabled across the grid. To disable a rule, add its Signature ID (SID) to the Current Grid Value box, one entry per line. To disable multiple rules, you can use regular expressions. + description: Contains the list of NIDS rules (or regex patterns) disabled across the grid. This setting is readonly; Use the Detections screen to disable rules. global: True multiline: True forcedType: "[]string" regex: \d*|re:.* helpLink: managing-alerts.html + readonlyUi: True + advanced: true enabled: - description: Contains the list of NIDS rules manually enabled across the grid. To enable a rule, add its Signature ID (SID) to the Current Grid Value box, one entry per line. To enable multiple rules, you can use regular expressions. + description: Contains the list of NIDS rules (or regex patterns) enabled across the grid. This setting is readonly; Use the Detections screen to enable rules. global: True multiline: True forcedType: "[]string" regex: \d*|re:.* helpLink: managing-alerts.html + readonlyUi: True + advanced: true modify: - description: Contains the list of NIDS rules that were modified from their default values. Entries must adhere to the following format - SID "REGEX_SEARCH_TERM" "REGEX_REPLACE_TERM" + description: Contains the list of NIDS rules (SID "REGEX_SEARCH_TERM" "REGEX_REPLACE_TERM"). This setting is readonly; Use the Detections screen to modify rules. global: True multiline: True forcedType: "[]string" helpLink: managing-alerts.html + readonlyUi: True + advanced: true rules: local__rules: - description: Contains the list of custom NIDS rules applied to the grid. To add custom NIDS rules to the grid, enter one rule per line in the Current Grid Value box. + description: Contains the list of custom NIDS rules applied to the grid. This setting is readonly; Use the Detections screen to adjust rules. file: True global: True advanced: True title: Local Rules helpLink: local-rules.html + readonlyUi: True filters__rules: description: If you are using Suricata for metadata, then you can set custom filters for that metadata here. file: True diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls index 64479e937..cdacfaa74 100644 --- a/salt/idstools/sync_files.sls +++ b/salt/idstools/sync_files.sls @@ -21,7 +21,7 @@ idstoolsetcsync: rulesdir: file.directory: - - name: /opt/so/rules/nids + - name: /opt/so/rules/nids/suri - user: 939 - group: 939 - makedirs: True @@ -29,7 +29,7 @@ rulesdir: # Don't show changes because all.rules can be large synclocalnidsrules: file.recurse: - - name: /opt/so/rules/nids/ + - name: /opt/so/rules/nids/suri/ - source: salt://idstools/rules/ - user: 939 - group: 939 diff --git a/salt/idstools/tools/sbin_jinja/so-rule-update b/salt/idstools/tools/sbin_jinja/so-rule-update index db110abc1..da4c272dd 100755 --- a/salt/idstools/tools/sbin_jinja/so-rule-update +++ b/salt/idstools/tools/sbin_jinja/so-rule-update @@ -26,8 +26,6 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force {%- elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %} docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }} -{%- elif IDSTOOLSMERGED.config.ruleset == 'TALOS' %} - docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --url=https://www.snort.org/rules/snortrules-snapshot-2983.tar.gz?oinkcode={{ IDSTOOLSMERGED.config.oinkcode }} {%- endif %} {%- endif %} diff --git a/salt/influxdb/templates/dashboard-security_onion_performance.json b/salt/influxdb/templates/dashboard-security_onion_performance.json index e4f2a6d38..4f543c8d1 100644 --- a/salt/influxdb/templates/dashboard-security_onion_performance.json +++ b/salt/influxdb/templates/dashboard-security_onion_performance.json @@ -1 +1 @@ -[{"apiVersion":"influxdata.com/v2alpha1","kind":"Dashboard","metadata":{"name":"vivid-wilson-002001"},"spec":{"charts":[{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Uptime","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime\")\n |> hostFilter()\n |> map(fn: (r) => ({r with _value: r._value / (24 * 60 * 60)}))\n |> group(columns: [\"host\"])\n |> last()\n |> lowestMin(n:1)"}],"staticLegend":{},"suffix":" days","width":1},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"},{"id":"z83MTSufTrlrCoEPiBXda","name":"ruby","type":"text","hex":"#BF3D5E","value":1}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Critical Alarms","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> filter(fn: (r) => r[\"_level\"] == \"crit\")\n |> count()"}],"staticLegend":{},"suffix":" ","width":1,"yPos":2},{"colors":[{"id":"base","name":"rainforest","type":"text","hex":"#4ED8A0"},{"id":"QCTYWuGuHkikYFsZSKMzQ","name":"rainforest","type":"text","hex":"#4ED8A0"},{"id":"QdpMyTRBb0LJ56-P5wfAW","name":"laser","type":"text","hex":"#00C9FF","value":1},{"id":"VQGwCoMrxZyP8asiOW5Cq","name":"tiger","type":"text","hex":"#F48D38","value":2},{"id":"zSO9QkesSIxrU_ntCBx2i","name":"ruby","type":"text","hex":"#BF3D5E","value":3}],"fieldOptions":[{"fieldName":"_time","visible":true},{"displayName":"Alarm","fieldName":"_check_name","visible":true},{"displayName":"Severity","fieldName":"_value","visible":true},{"displayName":"Status","fieldName":"_level","visible":true}],"height":6,"kind":"Table","name":"Alarm Status","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> drop(columns: [\"_value\"])\n |> duplicate(column: \"_level\", as: \"_value\")\n |> map(fn: (r) => ({ r with _value: if r._value == \"ok\" then 0 else if r._value == \"info\" then 1 else if r._value == \"warn\" then 2 else 3 }))\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> keep(columns: [\"_check_name\",\"_level\",\"_value\"])"}],"staticLegend":{},"tableOptions":{"sortBy":"_check_name","verticalTimeAxis":true},"timeFormat":"YYYY-MM-DD HH:mm:ss","width":3,"yPos":4},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"B"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Elasticsearch Storage Size","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"store_size_in_bytes\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"mean\")"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"store_size_in_bytes\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":10},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"B"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"InfluxDB Size","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"influxsize\")\n |> filter(fn: (r) => r[\"_field\"] == \"kbytes\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 1000.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"mean\")"},{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"influxsize\")\n |> filter(fn: (r) => r[\"_field\"] == \"kbytes\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 1000.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":14},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":" days"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"System Uptime","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24 * 60 * 60)}))\n |> yield(name: \"last\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24 * 60 * 60)}))\n |> yield(name: \"Trend\")"}],"shade":true,"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":18},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"System CPU Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_idle\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: r._value * -1.0 + 100.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_idle\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\",\"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: r._value * -1.0 + 100.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":22},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"QDwChKZWuQV0BaJcEeSam","name":"Atlantis","type":"scale","hex":"#74D495"},{"id":"ThD0WTqKHltQEVlq9mo6K","name":"Atlantis","type":"scale","hex":"#3F3FBA"},{"id":"FBHYZiwDLKyQK3eRfUD-0","name":"Atlantis","type":"scale","hex":"#FF4D9E"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"System Memory Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":26},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Monitor Interface Traffic - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"bytes_recv\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":30},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Management Interface Traffic - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"bytes_recv\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":6,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":34},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Stenographer Packet Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"stenodrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"stenodrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":38},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Disk Usage /","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":42},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"5m Load Average","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"load5\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"width":1,"xPos":1},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"},{"id":"z83MTSufTrlrCoEPiBXda","name":"tiger","type":"text","hex":"#F48D38","value":1}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Warning Alarms","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> filter(fn: (r) => r[\"_level\"] == \"warn\")\n |> count()"}],"staticLegend":{},"suffix":" ","width":1,"xPos":1,"yPos":2},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"IO Wait","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_iowait\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":2},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"},{"id":"z83MTSufTrlrCoEPiBXda","name":"laser","type":"text","hex":"#00C9FF","value":1}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Informative Alarms","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> filter(fn: (r) => r[\"_level\"] == \"info\")\n |> count()"}],"staticLegend":{},"suffix":" ","width":1,"xPos":2,"yPos":2},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Estimated EPS In","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"in\")\n |> hostFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"width":1,"xPos":3},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"pineapple","type":"threshold","hex":"#FFB94A","value":70},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"fire","type":"threshold","hex":"#DC4E58","value":80},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"CPU Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_idle\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> map(fn: (r) => ({r with _value: r._value * -1.0 + 100.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":3,"yPos":2},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"kOQLOg2H4FVEE-E1_L8Kq","name":"laser","type":"threshold","hex":"#00C9FF","value":85},{"id":"5IArg2lDb8KvnphywgUXa","name":"tiger","type":"threshold","hex":"#F48D38","value":90},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"ruby","type":"threshold","hex":"#BF3D5E","value":95},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"Root Disk Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"path\"] == \"/\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":3,"yPos":6},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Suricata Packet Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"suridrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"suridrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","xPos":3,"yCol":"_value","yPos":38},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Redis Queue","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"redisqueue\")\n |> filter(fn: (r) => r[\"_field\"] == \"unparsed\")\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"width":1,"xPos":4},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Elasticsearch Document Count","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"docs_count\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"mean\")"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"docs_count\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":10},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"heightRatio":0.301556420233463,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Redis Queue","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"redisqueue\")\n |> filter(fn: (r) => r[\"_field\"] == \"unparsed\")\n |> group(columns: [\"host\", \"_field\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"redisqueue\")\n |> filter(fn: (r) => r[\"_field\"] == \"unparsed\")\n |> group(columns: [\"host\", \"_field\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.301556420233463,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":14},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":" days"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container Uptime","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_status\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime_ns\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> group(columns: [\"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24 * 60 * 60 * 1000000000)}))\n |> yield(name: \"last\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_status\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime_ns\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> group(columns: [\"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24.0 * 60.0 * 60.0 * 1000000000.0)}))\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":18},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"yT5vTIlaaFChSrQvKLfqf","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"mzzUVSu3ibTph1JmQmDAQ","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"mOcnDo7l8ii6qNLFIB5rs","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container CPU Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"mean\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":22},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"QDwChKZWuQV0BaJcEeSam","name":"Atlantis","type":"scale","hex":"#74D495"},{"id":"ThD0WTqKHltQEVlq9mo6K","name":"Atlantis","type":"scale","hex":"#3F3FBA"},{"id":"FBHYZiwDLKyQK3eRfUD-0","name":"Atlantis","type":"scale","hex":"#FF4D9E"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container Memory Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"mean\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":26},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b"}],"colorizeRows":true,"colors":[{"id":"0ynR6Zs0wuQ3WY0Lz-_KC","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"YiArehCNBwFm9mn8DSXSG","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"DxByY_EQW9Xs2jD5ktkG5","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container Traffic - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_net\")\n |> filter(fn: (r) => r[\"_field\"] == \"rx_bytes\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"mean\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_net\")\n |> filter(fn: (r) => r[\"_field\"] == \"rx_bytes\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with _value: r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":30},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Disk Usage /nsm","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/nsm\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/nsm\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xPos":4,"yPos":42},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Inbound Traffic","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\") \n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: r._value * 8.0 / (1000.0 * 1000.0)}))\n |> group(columns: [\"host\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> last()\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":" Mb/s","width":1,"xPos":5},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Inbound Drops","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop_in\") \n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: r._value * 8.0 / (1000.0 * 1000.0)}))\n |> group(columns: [\"host\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> last()\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":" Mb/s","width":1,"xPos":6},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"pineapple","type":"threshold","hex":"#FFB94A","value":70},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"fire","type":"threshold","hex":"#DC4E58","value":80},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"Memory Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":6,"yPos":2},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"laser","type":"threshold","hex":"#00C9FF","value":85},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"tiger","type":"threshold","hex":"#F48D38","value":90},{"id":"H7uprvKmMEh39en6X-ms_","name":"ruby","type":"threshold","hex":"#BF3D5E","value":95},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"NSM Disk Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"path\"] == \"/nsm\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":6,"yPos":6},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Management Interface Traffic - Outbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_sent\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n \n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"bytes_sent\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_sent\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n \n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":6,"widthRatio":1,"xCol":"_time","xPos":6,"yCol":"_value","yPos":34},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Zeek Packet Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekdrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekdrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","xPos":6,"yCol":"_value","yPos":38},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Capture Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekcaptureloss\")\n |> filter(fn: (r) => r[\"_field\"] == \"loss\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":7},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Zeek Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekdrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":8},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"s"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"heightRatio":0.301556420233463,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Elastic Ingest Time Spent","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_community_id_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"community.id_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_conditional_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"conditional_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_date_index_name_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"date.index.name_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_date_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"date_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_dissect_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"dissect_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_dot_expander_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"dot.expander_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_geoip_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"geoip_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_grok_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"grok_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_json_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"json_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_kv_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"kv_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_lowercase_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"lowercase_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_rename_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"rename_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_script_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"script_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_user_agent_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"user.agent_time\")"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.301556420233463,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":10},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"1m Load Average","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"load1\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"load1\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\",\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":14,"yTickStep":1},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":" e/s"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"heightRatio":0.301556420233463,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Logstash EPS","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"in\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"out\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: -r._value}))\n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"in\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"out\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: -r._value}))\n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.301556420233463,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":18},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"UAehjIsi65P8u92M_3sQY","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"_SCP8Npp4NVMx2N4mfuzX","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"BoMPg4R1KDp_UsRORdV3_","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"IO Wait","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_iowait\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_iowait\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":22},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"QDwChKZWuQV0BaJcEeSam","name":"Atlantis","type":"scale","hex":"#74D495"},{"id":"ThD0WTqKHltQEVlq9mo6K","name":"Atlantis","type":"scale","hex":"#3F3FBA"},{"id":"FBHYZiwDLKyQK3eRfUD-0","name":"Atlantis","type":"scale","hex":"#FF4D9E"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Swap Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"swap\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"swap\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":26},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Monitor Interface Drops - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop_in\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"drop_in\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop_in\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":30},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":" days"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Stenographer PCAP Retention","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"pcapage\")\n |> filter(fn: (r) => r[\"_field\"] == \"seconds\")\n |> map(fn: (r) => ({ r with _value: r._value / (24.0 * 3600.0)}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])"},{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"pcapage\")\n |> filter(fn: (r) => r[\"_field\"] == \"seconds\")\n |> map(fn: (r) => ({ r with _value: r._value / (24.0 * 3600.0)}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":42},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Suricata Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"suridrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":9},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"pineapple","type":"threshold","hex":"#FFB94A","value":50},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"fire","type":"threshold","hex":"#DC4E58","value":70},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"Swap Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"swap\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":9,"yPos":2},{"colors":[{"id":"base","name":"white","type":"text","hex":"#ffffff"}],"fieldOptions":[{"displayName":"Host","fieldName":"host","visible":true},{"displayName":"Name","fieldName":"container_name","visible":true},{"displayName":"Status","fieldName":"container_status","visible":true},{"displayName":"OOM Killed","fieldName":"_value","visible":true},{"displayName":"_start","fieldName":"_start","visible":true},{"displayName":"_stop","fieldName":"_stop","visible":true},{"displayName":"_time","fieldName":"_time","visible":true},{"displayName":"_field","fieldName":"_field","visible":true},{"displayName":"_measurement","fieldName":"_measurement","visible":true},{"displayName":"engine_host","fieldName":"engine_host","visible":true},{"displayName":"role","fieldName":"role","visible":true},{"displayName":"server_version","fieldName":"server_version","visible":true},{"displayName":"container_image","fieldName":"container_image","visible":true},{"displayName":"container_version","fieldName":"container_version","visible":true},{"displayName":"description","fieldName":"description","visible":true},{"displayName":"maintainer","fieldName":"maintainer","visible":true},{"displayName":"io.k8s.description","fieldName":"io.k8s.description","visible":true},{"displayName":"io.k8s.display-name","fieldName":"io.k8s.display-name","visible":true},{"displayName":"license","fieldName":"license","visible":true},{"displayName":"name","fieldName":"name","visible":true},{"displayName":"org.label-schema.build-date","fieldName":"org.label-schema.build-date","visible":true},{"displayName":"org.label-schema.license","fieldName":"org.label-schema.license","visible":true},{"displayName":"org.label-schema.name","fieldName":"org.label-schema.name","visible":true},{"displayName":"org.label-schema.schema-version","fieldName":"org.label-schema.schema-version","visible":true},{"displayName":"org.label-schema.url","fieldName":"org.label-schema.url","visible":true},{"displayName":"org.label-schema.vcs-ref","fieldName":"org.label-schema.vcs-ref","visible":true},{"displayName":"org.label-schema.vcs-url","fieldName":"org.label-schema.vcs-url","visible":true},{"displayName":"org.label-schema.vendor","fieldName":"org.label-schema.vendor","visible":true},{"displayName":"org.label-schema.version","fieldName":"org.label-schema.version","visible":true},{"displayName":"org.opencontainers.image.created","fieldName":"org.opencontainers.image.created","visible":true},{"displayName":"org.opencontainers.image.licenses","fieldName":"org.opencontainers.image.licenses","visible":true},{"displayName":"org.opencontainers.image.title","fieldName":"org.opencontainers.image.title","visible":true},{"displayName":"org.opencontainers.image.vendor","fieldName":"org.opencontainers.image.vendor","visible":true},{"displayName":"release","fieldName":"release","visible":true},{"displayName":"summary","fieldName":"summary","visible":true},{"displayName":"url","fieldName":"url","visible":true},{"displayName":"vendor","fieldName":"vendor","visible":true},{"displayName":"version","fieldName":"version","visible":true},{"displayName":"org.label-schema.usage","fieldName":"org.label-schema.usage","visible":true},{"displayName":"org.opencontainers.image.documentation","fieldName":"org.opencontainers.image.documentation","visible":true},{"displayName":"org.opencontainers.image.revision","fieldName":"org.opencontainers.image.revision","visible":true},{"displayName":"org.opencontainers.image.source","fieldName":"org.opencontainers.image.source","visible":true},{"displayName":"org.opencontainers.image.url","fieldName":"org.opencontainers.image.url","visible":true},{"displayName":"org.opencontainers.image.version","fieldName":"org.opencontainers.image.version","visible":true},{"displayName":"org.opencontainers.image.description","fieldName":"org.opencontainers.image.description","visible":true}],"height":4,"kind":"Table","name":"Most Recent Container Events","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_status\")\n |> filter(fn: (r) => r[\"_field\"] == \"oomkilled\")\n |> filter(fn: (r) => r[\"container_status\"] != \"running\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"container_name\", \"host\"])\n |> last()\n |> group()\n |> keep(columns: [\"_value\", \"container_name\", \"host\", \"container_status\"])"}],"staticLegend":{},"tableOptions":{"sortBy":"container_name","verticalTimeAxis":true},"timeFormat":"YYYY-MM-DD HH:mm:ss","width":3,"xPos":9,"yPos":6},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Zeek Capture Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekcaptureloss\")\n |> filter(fn: (r) => r[\"_field\"] == \"loss\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekcaptureloss\")\n |> filter(fn: (r) => r[\"_field\"] == \"loss\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","xPos":9,"yCol":"_value","yPos":38},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Stenographer Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"stenodrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":10},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"PCAP Retention","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"pcapage\")\n |> filter(fn: (r) => r[\"_field\"] == \"seconds\")\n |> hostFilter()\n |> map(fn: (r) => ({r with _value: r._value / (24.0 * 60.0 * 60.0)}))\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":" days","width":1,"xPos":11}],"description":"Visualize the Security Onion grid performance metrics and alarm statuses.","name":"Security Onion Performance"}}] \ No newline at end of file +[{"apiVersion":"influxdata.com/v2alpha1","kind":"Dashboard","metadata":{"name":"vivid-wilson-002001"},"spec":{"charts":[{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Uptime","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime\")\n |> hostFilter()\n |> map(fn: (r) => ({r with _value: r._value / (24 * 60 * 60)}))\n |> group(columns: [\"host\"])\n |> last()\n |> lowestMin(n:1)"}],"staticLegend":{},"suffix":" days","width":1},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"},{"id":"z83MTSufTrlrCoEPiBXda","name":"ruby","type":"text","hex":"#BF3D5E","value":1}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Critical Alarms","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> filter(fn: (r) => r[\"_level\"] == \"crit\")\n |> count()"}],"staticLegend":{},"suffix":" ","width":1,"yPos":2},{"colors":[{"id":"base","name":"rainforest","type":"text","hex":"#4ED8A0"},{"id":"QCTYWuGuHkikYFsZSKMzQ","name":"rainforest","type":"text","hex":"#4ED8A0"},{"id":"QdpMyTRBb0LJ56-P5wfAW","name":"laser","type":"text","hex":"#00C9FF","value":1},{"id":"VQGwCoMrxZyP8asiOW5Cq","name":"tiger","type":"text","hex":"#F48D38","value":2},{"id":"zSO9QkesSIxrU_ntCBx2i","name":"ruby","type":"text","hex":"#BF3D5E","value":3}],"fieldOptions":[{"fieldName":"_time","visible":true},{"displayName":"Alarm","fieldName":"_check_name","visible":true},{"displayName":"Severity","fieldName":"_value","visible":true},{"displayName":"Status","fieldName":"_level","visible":true}],"height":6,"kind":"Table","name":"Alarm Status","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> drop(columns: [\"_value\"])\n |> duplicate(column: \"_level\", as: \"_value\")\n |> map(fn: (r) => ({ r with _value: if r._value == \"ok\" then 0 else if r._value == \"info\" then 1 else if r._value == \"warn\" then 2 else 3 }))\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> keep(columns: [\"_check_name\",\"_level\",\"_value\"])"}],"staticLegend":{},"tableOptions":{"sortBy":"_check_name","verticalTimeAxis":true},"timeFormat":"YYYY-MM-DD HH:mm:ss","width":3,"yPos":4},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"B"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Elasticsearch Storage Size","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"store_size_in_bytes\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"mean\")"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"store_size_in_bytes\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":10},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"B"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"InfluxDB Size","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"influxsize\")\n |> filter(fn: (r) => r[\"_field\"] == \"kbytes\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 1000.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"mean\")"},{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"influxsize\")\n |> filter(fn: (r) => r[\"_field\"] == \"kbytes\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 1000.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":14},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":" days"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"System Uptime","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24 * 60 * 60)}))\n |> yield(name: \"last\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24 * 60 * 60)}))\n |> yield(name: \"Trend\")"}],"shade":true,"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":18},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"lQ75rvTyd2Lq5pZjzy6LB","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"KLfpRZtiEnU2GxjPtrrzQ","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"1kLynwKxvJ3B5IeJnrBqp","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Kafka EPS","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_topics\")\n |> filter(fn: (r) => r[\"_field\"] == \"MessagesInPerSec.Count\")\n |> derivative(unit: 1s, nonNegative: true)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_topics\")\n |> filter(fn: (r) => r[\"_field\"] == \"MessagesInPerSec.Count\")\n |> derivative(unit: 1s, nonNegative: true)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":22},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"System CPU Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_idle\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: r._value * -1.0 + 100.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_idle\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\",\"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: r._value * -1.0 + 100.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":26},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"QDwChKZWuQV0BaJcEeSam","name":"Atlantis","type":"scale","hex":"#74D495"},{"id":"ThD0WTqKHltQEVlq9mo6K","name":"Atlantis","type":"scale","hex":"#3F3FBA"},{"id":"FBHYZiwDLKyQK3eRfUD-0","name":"Atlantis","type":"scale","hex":"#FF4D9E"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"System Memory Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":30},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Monitor Interface Traffic - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"bytes_recv\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":34},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Management Interface Traffic - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"bytes_recv\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":6,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":38},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Stenographer Packet Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"stenodrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"stenodrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":42},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Disk Usage /","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":46},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"5m Load Average","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"load5\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"width":1,"xPos":1},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"},{"id":"z83MTSufTrlrCoEPiBXda","name":"tiger","type":"text","hex":"#F48D38","value":1}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Warning Alarms","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> filter(fn: (r) => r[\"_level\"] == \"warn\")\n |> count()"}],"staticLegend":{},"suffix":" ","width":1,"xPos":1,"yPos":2},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"IO Wait","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_iowait\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":2},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"},{"id":"z83MTSufTrlrCoEPiBXda","name":"laser","type":"text","hex":"#00C9FF","value":1}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Informative Alarms","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> filter(fn: (r) => r[\"_level\"] == \"info\")\n |> count()"}],"staticLegend":{},"suffix":" ","width":1,"xPos":2,"yPos":2},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Estimated EPS In","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"in\")\n |> hostFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"width":1,"xPos":3},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"pineapple","type":"threshold","hex":"#FFB94A","value":70},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"fire","type":"threshold","hex":"#DC4E58","value":80},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"CPU Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_idle\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> map(fn: (r) => ({r with _value: r._value * -1.0 + 100.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":3,"yPos":2},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"kOQLOg2H4FVEE-E1_L8Kq","name":"laser","type":"threshold","hex":"#00C9FF","value":85},{"id":"5IArg2lDb8KvnphywgUXa","name":"tiger","type":"threshold","hex":"#F48D38","value":90},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"ruby","type":"threshold","hex":"#BF3D5E","value":95},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"Root Disk Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"path\"] == \"/\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":3,"yPos":6},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Suricata Packet Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"suridrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"suridrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","xPos":3,"yCol":"_value","yPos":42},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Redis Queue","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"redisqueue\")\n |> filter(fn: (r) => r[\"_field\"] == \"unparsed\")\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"width":1,"xPos":4},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Elasticsearch Document Count","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"docs_count\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"mean\")"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"docs_count\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":10},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"heightRatio":0.301556420233463,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Redis Queue","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"redisqueue\")\n |> filter(fn: (r) => r[\"_field\"] == \"unparsed\")\n |> group(columns: [\"host\", \"_field\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"redisqueue\")\n |> filter(fn: (r) => r[\"_field\"] == \"unparsed\")\n |> group(columns: [\"host\", \"_field\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.301556420233463,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":14},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":" days"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container Uptime","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_status\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime_ns\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> group(columns: [\"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24 * 60 * 60 * 1000000000)}))\n |> yield(name: \"last\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_status\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime_ns\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> group(columns: [\"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24.0 * 60.0 * 60.0 * 1000000000.0)}))\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":18},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":2,"hoverDimension":"auto","kind":"Single_Stat_Plus_Line","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Kafka Active Controllers","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_controller\")\n |> filter(fn: (r) => r[\"_field\"] == \"ActiveControllerCount.Value\")\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"current\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_controller\")\n |> filter(fn: (r) => r[\"_field\"] == \"ActiveControllerCount.Value\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":22},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":2,"hoverDimension":"auto","kind":"Single_Stat_Plus_Line","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Kafka Active Brokers","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_controller\")\n |> filter(fn: (r) => r[\"_field\"] == \"ActiveBrokerCount.Value\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"trend\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_controller\")\n |> filter(fn: (r) => r[\"_field\"] == \"ActiveBrokerCount.Value\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"current\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":24},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"yT5vTIlaaFChSrQvKLfqf","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"mzzUVSu3ibTph1JmQmDAQ","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"mOcnDo7l8ii6qNLFIB5rs","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container CPU Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"mean\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":26},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"QDwChKZWuQV0BaJcEeSam","name":"Atlantis","type":"scale","hex":"#74D495"},{"id":"ThD0WTqKHltQEVlq9mo6K","name":"Atlantis","type":"scale","hex":"#3F3FBA"},{"id":"FBHYZiwDLKyQK3eRfUD-0","name":"Atlantis","type":"scale","hex":"#FF4D9E"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container Memory Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"mean\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":30},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b"}],"colorizeRows":true,"colors":[{"id":"0ynR6Zs0wuQ3WY0Lz-_KC","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"YiArehCNBwFm9mn8DSXSG","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"DxByY_EQW9Xs2jD5ktkG5","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container Traffic - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_net\")\n |> filter(fn: (r) => r[\"_field\"] == \"rx_bytes\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"mean\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_net\")\n |> filter(fn: (r) => r[\"_field\"] == \"rx_bytes\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with _value: r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":34},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Disk Usage /nsm","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/nsm\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/nsm\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xPos":4,"yPos":46},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Inbound Traffic","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\") \n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: r._value * 8.0 / (1000.0 * 1000.0)}))\n |> group(columns: [\"host\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> last()\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":" Mb/s","width":1,"xPos":5},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Inbound Drops","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop_in\") \n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: r._value * 8.0 / (1000.0 * 1000.0)}))\n |> group(columns: [\"host\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> last()\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":" Mb/s","width":1,"xPos":6},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"pineapple","type":"threshold","hex":"#FFB94A","value":70},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"fire","type":"threshold","hex":"#DC4E58","value":80},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"Memory Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":6,"yPos":2},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"laser","type":"threshold","hex":"#00C9FF","value":85},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"tiger","type":"threshold","hex":"#F48D38","value":90},{"id":"H7uprvKmMEh39en6X-ms_","name":"ruby","type":"threshold","hex":"#BF3D5E","value":95},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"NSM Disk Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"path\"] == \"/nsm\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":6,"yPos":6},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Management Interface Traffic - Outbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_sent\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n \n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"bytes_sent\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_sent\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n \n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":6,"widthRatio":1,"xCol":"_time","xPos":6,"yCol":"_value","yPos":38},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Zeek Packet Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekdrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekdrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","xPos":6,"yCol":"_value","yPos":42},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Capture Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekcaptureloss\")\n |> filter(fn: (r) => r[\"_field\"] == \"loss\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":7},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Zeek Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekdrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":8},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"s"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"heightRatio":0.301556420233463,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Elastic Ingest Time Spent","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_community_id_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"community.id_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_conditional_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"conditional_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_date_index_name_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"date.index.name_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_date_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"date_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_dissect_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"dissect_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_dot_expander_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"dot.expander_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_geoip_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"geoip_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_grok_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"grok_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_json_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"json_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_kv_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"kv_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_lowercase_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"lowercase_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_rename_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"rename_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_script_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"script_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_user_agent_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"user.agent_time\")"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.301556420233463,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":10},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"1m Load Average","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"load1\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"load1\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\",\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":14,"yTickStep":1},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":" e/s"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"heightRatio":0.301556420233463,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Logstash EPS","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"in\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"out\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: -r._value}))\n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"in\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"out\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: -r._value}))\n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.301556420233463,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":18},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":4,"hoverDimension":"auto","kind":"Single_Stat_Plus_Line","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Kafka Under Replicated Partitions","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_partition\")\n |> filter(fn: (r) => r[\"_field\"] == \"UnderReplicatedPartitions\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"partition\",\"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_partition\")\n |> filter(fn: (r) => r[\"_field\"] == \"UnderReplicatedPartitions\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"partition\",\"host\", \"role\"])\n |> yield(name: \"trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":22},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"UAehjIsi65P8u92M_3sQY","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"_SCP8Npp4NVMx2N4mfuzX","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"BoMPg4R1KDp_UsRORdV3_","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"IO Wait","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_iowait\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_iowait\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":26},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"QDwChKZWuQV0BaJcEeSam","name":"Atlantis","type":"scale","hex":"#74D495"},{"id":"ThD0WTqKHltQEVlq9mo6K","name":"Atlantis","type":"scale","hex":"#3F3FBA"},{"id":"FBHYZiwDLKyQK3eRfUD-0","name":"Atlantis","type":"scale","hex":"#FF4D9E"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Swap Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"swap\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"swap\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":30},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Monitor Interface Drops - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop_in\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"drop_in\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop_in\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":34},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":" days"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Stenographer PCAP Retention","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"pcapage\")\n |> filter(fn: (r) => r[\"_field\"] == \"seconds\")\n |> map(fn: (r) => ({ r with _value: r._value / (24.0 * 3600.0)}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])"},{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"pcapage\")\n |> filter(fn: (r) => r[\"_field\"] == \"seconds\")\n |> map(fn: (r) => ({ r with _value: r._value / (24.0 * 3600.0)}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":46},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Suricata Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"suridrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":9},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"pineapple","type":"threshold","hex":"#FFB94A","value":50},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"fire","type":"threshold","hex":"#DC4E58","value":70},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"Swap Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"swap\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":9,"yPos":2},{"colors":[{"id":"base","name":"white","type":"text","hex":"#ffffff"}],"fieldOptions":[{"displayName":"Host","fieldName":"host","visible":true},{"displayName":"Name","fieldName":"container_name","visible":true},{"displayName":"Status","fieldName":"container_status","visible":true},{"displayName":"OOM Killed","fieldName":"_value","visible":true},{"displayName":"_start","fieldName":"_start","visible":true},{"displayName":"_stop","fieldName":"_stop","visible":true},{"displayName":"_time","fieldName":"_time","visible":true},{"displayName":"_field","fieldName":"_field","visible":true},{"displayName":"_measurement","fieldName":"_measurement","visible":true},{"displayName":"engine_host","fieldName":"engine_host","visible":true},{"displayName":"role","fieldName":"role","visible":true},{"displayName":"server_version","fieldName":"server_version","visible":true},{"displayName":"container_image","fieldName":"container_image","visible":true},{"displayName":"container_version","fieldName":"container_version","visible":true},{"displayName":"description","fieldName":"description","visible":true},{"displayName":"maintainer","fieldName":"maintainer","visible":true},{"displayName":"io.k8s.description","fieldName":"io.k8s.description","visible":true},{"displayName":"io.k8s.display-name","fieldName":"io.k8s.display-name","visible":true},{"displayName":"license","fieldName":"license","visible":true},{"displayName":"name","fieldName":"name","visible":true},{"displayName":"org.label-schema.build-date","fieldName":"org.label-schema.build-date","visible":true},{"displayName":"org.label-schema.license","fieldName":"org.label-schema.license","visible":true},{"displayName":"org.label-schema.name","fieldName":"org.label-schema.name","visible":true},{"displayName":"org.label-schema.schema-version","fieldName":"org.label-schema.schema-version","visible":true},{"displayName":"org.label-schema.url","fieldName":"org.label-schema.url","visible":true},{"displayName":"org.label-schema.vcs-ref","fieldName":"org.label-schema.vcs-ref","visible":true},{"displayName":"org.label-schema.vcs-url","fieldName":"org.label-schema.vcs-url","visible":true},{"displayName":"org.label-schema.vendor","fieldName":"org.label-schema.vendor","visible":true},{"displayName":"org.label-schema.version","fieldName":"org.label-schema.version","visible":true},{"displayName":"org.opencontainers.image.created","fieldName":"org.opencontainers.image.created","visible":true},{"displayName":"org.opencontainers.image.licenses","fieldName":"org.opencontainers.image.licenses","visible":true},{"displayName":"org.opencontainers.image.title","fieldName":"org.opencontainers.image.title","visible":true},{"displayName":"org.opencontainers.image.vendor","fieldName":"org.opencontainers.image.vendor","visible":true},{"displayName":"release","fieldName":"release","visible":true},{"displayName":"summary","fieldName":"summary","visible":true},{"displayName":"url","fieldName":"url","visible":true},{"displayName":"vendor","fieldName":"vendor","visible":true},{"displayName":"version","fieldName":"version","visible":true},{"displayName":"org.label-schema.usage","fieldName":"org.label-schema.usage","visible":true},{"displayName":"org.opencontainers.image.documentation","fieldName":"org.opencontainers.image.documentation","visible":true},{"displayName":"org.opencontainers.image.revision","fieldName":"org.opencontainers.image.revision","visible":true},{"displayName":"org.opencontainers.image.source","fieldName":"org.opencontainers.image.source","visible":true},{"displayName":"org.opencontainers.image.url","fieldName":"org.opencontainers.image.url","visible":true},{"displayName":"org.opencontainers.image.version","fieldName":"org.opencontainers.image.version","visible":true},{"displayName":"org.opencontainers.image.description","fieldName":"org.opencontainers.image.description","visible":true}],"height":4,"kind":"Table","name":"Most Recent Container Events","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_status\")\n |> filter(fn: (r) => r[\"_field\"] == \"oomkilled\")\n |> filter(fn: (r) => r[\"container_status\"] != \"running\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"container_name\", \"host\"])\n |> last()\n |> group()\n |> keep(columns: [\"_value\", \"container_name\", \"host\", \"container_status\"])"}],"staticLegend":{},"tableOptions":{"sortBy":"container_name","verticalTimeAxis":true},"timeFormat":"YYYY-MM-DD HH:mm:ss","width":3,"xPos":9,"yPos":6},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Zeek Capture Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekcaptureloss\")\n |> filter(fn: (r) => r[\"_field\"] == \"loss\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekcaptureloss\")\n |> filter(fn: (r) => r[\"_field\"] == \"loss\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","xPos":9,"yCol":"_value","yPos":42},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Stenographer Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"stenodrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":10},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"PCAP Retention","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"pcapage\")\n |> filter(fn: (r) => r[\"_field\"] == \"seconds\")\n |> hostFilter()\n |> map(fn: (r) => ({r with _value: r._value / (24.0 * 60.0 * 60.0)}))\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":" days","width":1,"xPos":11}],"description":"Visualize the Security Onion grid performance metrics and alarm statuses.","name":"Security Onion Performance"}}] \ No newline at end of file diff --git a/salt/kafka/config.map.jinja b/salt/kafka/config.map.jinja new file mode 100644 index 000000000..4c408a1e7 --- /dev/null +++ b/salt/kafka/config.map.jinja @@ -0,0 +1,91 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} +{% from 'kafka/map.jinja' import KAFKAMERGED %} +{% from 'vars/globals.map.jinja' import GLOBALS %} + +{% set KAFKA_NODES_PILLAR = salt['pillar.get']('kafka:nodes') %} +{% set KAFKA_PASSWORD = salt['pillar.get']('kafka:password') %} + +{# Create list of KRaft controllers #} +{% set controllers = [] %} + +{# Check for Kafka nodes with controller in process_x_roles #} +{% for node in KAFKA_NODES_PILLAR %} +{% if 'controller' in KAFKA_NODES_PILLAR[node].role %} +{% do controllers.append(KAFKA_NODES_PILLAR[node].nodeid ~ "@" ~ node ~ ":9093") %} +{% endif %} +{% endfor %} + +{% set kafka_controller_quorum_voters = ','.join(controllers) %} + +{# By default all Kafka eligible nodes are given the role of broker, except for + grid MANAGER (broker,controller) until overridden through SOC UI #} +{% set node_type = salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname + ':role') %} + +{# Generate server.properties for 'broker' , 'controller', 'broker,controller' node types + anything above this line is a configuration needed for ALL Kafka nodes #} +{% if node_type == 'broker' %} +{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %} +{% do KAFKAMERGED.config.broker.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %} +{% do KAFKAMERGED.config.broker.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %} +{% do KAFKAMERGED.config.broker.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %} + +{# Nodes with only the 'broker' role need to have the below settings for communicating with controller nodes #} +{% do KAFKAMERGED.config.broker.update({'controller_x_listener_x_names': KAFKAMERGED.config.controller.controller_x_listener_x_names }) %} +{% do KAFKAMERGED.config.broker.update({ + 'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map + + ',' + KAFKAMERGED.config.controller.listener_x_security_x_protocol_x_map }) + %} +{% endif %} + +{% if node_type == 'controller' %} +{% do KAFKAMERGED.config.controller.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %} +{% do KAFKAMERGED.config.controller.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %} +{% do KAFKAMERGED.config.controller.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %} + +{% endif %} + +{# Kafka nodes of this type are not recommended for use outside of development / testing. #} +{% if node_type == 'broker,controller' %} +{% do KAFKAMERGED.config.broker.update({'advertised_x_listeners': 'BROKER://'+ GLOBALS.node_ip +':9092' }) %} +{% do KAFKAMERGED.config.broker.update({'controller_x_listener_x_names': KAFKAMERGED.config.controller.controller_x_listener_x_names }) %} +{% do KAFKAMERGED.config.broker.update({'controller_x_quorum_x_voters': kafka_controller_quorum_voters }) %} +{% do KAFKAMERGED.config.broker.update({'node_x_id': salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname +':nodeid') }) %} +{% do KAFKAMERGED.config.broker.update({'process_x_roles': 'broker,controller' }) %} +{% do KAFKAMERGED.config.broker.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %} + +{% do KAFKAMERGED.config.broker.update({ + 'listeners': KAFKAMERGED.config.broker.listeners + ',' + KAFKAMERGED.config.controller.listeners }) + %} + +{% do KAFKAMERGED.config.broker.update({ + 'listener_x_security_x_protocol_x_map': KAFKAMERGED.config.broker.listener_x_security_x_protocol_x_map + + ',' + KAFKAMERGED.config.controller.listener_x_security_x_protocol_x_map }) + %} + +{% endif %} + +{# If a password other than PLACEHOLDER isn't set remove it from the server.properties #} +{% if KAFKAMERGED.config.broker.ssl_x_truststore_x_password == 'PLACEHOLDER' %} +{% do KAFKAMERGED.config.broker.pop('ssl_x_truststore_x_password') %} +{% endif %} + +{% if KAFKAMERGED.config.controller.ssl_x_truststore_x_password == 'PLACEHOLDER' %} +{% do KAFKAMERGED.config.controller.pop('ssl_x_truststore_x_password') %} +{% endif %} + +{# Client properties stuff #} +{% if KAFKAMERGED.config.client.ssl_x_truststore_x_password == 'PLACEHOLDER' %} +{% do KAFKAMERGED.config.client.pop('ssl_x_truststore_x_password') %} +{% endif %} +{% do KAFKAMERGED.config.client.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %} + +{% if 'broker' in node_type %} +{% set KAFKACONFIG = KAFKAMERGED.config.broker %} +{% else %} +{% set KAFKACONFIG = KAFKAMERGED.config.controller %} +{% endif %} + +{% set KAFKACLIENT = KAFKAMERGED.config.client %} \ No newline at end of file diff --git a/salt/kafka/config.sls b/salt/kafka/config.sls new file mode 100644 index 000000000..6293ee697 --- /dev/null +++ b/salt/kafka/config.sls @@ -0,0 +1,69 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} + +include: + - ssl + +kafka_group: + group.present: + - name: kafka + - gid: 960 + +kafka: + user.present: + - uid: 960 + - gid: 960 + +kafka_sbin_tools: + file.recurse: + - name: /usr/sbin + - source: salt://kafka/tools/sbin + - user: 960 + - group: 960 + - file_mode: 755 + +kafka_log_dir: + file.directory: + - name: /opt/so/log/kafka + - user: 960 + - group: 960 + - makedirs: True + +kafka_data_dir: + file.directory: + - name: /nsm/kafka/data + - user: 960 + - group: 960 + - makedirs: True + +{% for sc in ['server', 'client'] %} +kafka_kraft_{{sc}}_properties: + file.managed: + - source: salt://kafka/etc/{{sc}}.properties.jinja + - name: /opt/so/conf/kafka/{{sc}}.properties + - template: jinja + - user: 960 + - group: 960 + - makedirs: True + - show_changes: False +{% endfor %} + +reset_quorum_on_changes: + cmd.run: + - name: rm -f /nsm/kafka/data/__cluster_metadata-0/quorum-state + - onchanges: + - file: /opt/so/conf/kafka/server.properties + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/kafka/defaults.yaml b/salt/kafka/defaults.yaml new file mode 100644 index 000000000..ad626458f --- /dev/null +++ b/salt/kafka/defaults.yaml @@ -0,0 +1,62 @@ +kafka: + enabled: False + cluster_id: + password: + controllers: + reset: + config: + broker: + advertised_x_listeners: + auto_x_create_x_topics_x_enable: true + controller_x_quorum_x_voters: + default_x_replication_x_factor: 1 + inter_x_broker_x_listener_x_name: BROKER + listeners: BROKER://0.0.0.0:9092 + listener_x_security_x_protocol_x_map: BROKER:SSL + log_x_dirs: /nsm/kafka/data + log_x_retention_x_check_x_interval_x_ms: 300000 + log_x_retention_x_hours: 168 + log_x_segment_x_bytes: 1073741824 + node_x_id: + num_x_io_x_threads: 8 + num_x_network_x_threads: 3 + num_x_partitions: 3 + num_x_recovery_x_threads_x_per_x_data_x_dir: 1 + offsets_x_topic_x_replication_x_factor: 1 + process_x_roles: broker + socket_x_receive_x_buffer_x_bytes: 102400 + socket_x_request_x_max_x_bytes: 104857600 + socket_x_send_x_buffer_x_bytes: 102400 + ssl_x_keystore_x_location: /etc/pki/kafka.p12 + ssl_x_keystore_x_type: PKCS12 + ssl_x_keystore_x_password: + ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts + ssl_x_truststore_x_password: PLACEHOLDER + ssl_x_truststore_x_type: PEM + transaction_x_state_x_log_x_min_x_isr: 1 + transaction_x_state_x_log_x_replication_x_factor: 1 + client: + security_x_protocol: SSL + ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts + ssl_x_truststore_x_password: PLACEHOLDER + ssl_x_truststore_x_type: PEM + ssl_x_keystore_x_location: /etc/pki/kafka.p12 + ssl_x_keystore_x_type: PKCS12 + ssl_x_keystore_x_password: + controller: + controller_x_listener_x_names: CONTROLLER + controller_x_quorum_x_voters: + listeners: CONTROLLER://0.0.0.0:9093 + listener_x_security_x_protocol_x_map: CONTROLLER:SSL + log_x_dirs: /nsm/kafka/data + log_x_retention_x_check_x_interval_x_ms: 300000 + log_x_retention_x_hours: 168 + log_x_segment_x_bytes: 1073741824 + node_x_id: + process_x_roles: controller + ssl_x_keystore_x_location: /etc/pki/kafka.p12 + ssl_x_keystore_x_type: PKCS12 + ssl_x_keystore_x_password: + ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts + ssl_x_truststore_x_password: PLACEHOLDER + ssl_x_truststore_x_type: PEM \ No newline at end of file diff --git a/salt/kafka/disabled.sls b/salt/kafka/disabled.sls new file mode 100644 index 000000000..707e953a4 --- /dev/null +++ b/salt/kafka/disabled.sls @@ -0,0 +1,25 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'kafka/map.jinja' import KAFKAMERGED %} +{% from 'vars/globals.map.jinja' import GLOBALS %} + +so-kafka: + docker_container.absent: + - force: True + +so-kafka_so-status.disabled: + file.comment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-kafka$ + - onlyif: grep -q '^so-kafka$' /opt/so/conf/so-status/so-status.conf + +{% if GLOBALS.is_manager and KAFKAMERGED.enabled or GLOBALS.pipeline == "KAFKA" %} +ensure_default_pipeline: + cmd.run: + - name: | + /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False; + /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/global/soc_global.sls global.pipeline REDIS +{% endif %} \ No newline at end of file diff --git a/salt/kafka/enabled.sls b/salt/kafka/enabled.sls new file mode 100644 index 000000000..0837b5af6 --- /dev/null +++ b/salt/kafka/enabled.sls @@ -0,0 +1,86 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. +# +# Note: Per the Elastic License 2.0, the second limitation states: +# +# "You may not move, change, disable, or circumvent the license key functionality +# in the software, and you may not remove or obscure any functionality in the +# software that is protected by the license key." + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'docker/docker.map.jinja' import DOCKER %} +{% set KAFKANODES = salt['pillar.get']('kafka:nodes') %} +{% if 'gmd' in salt['pillar.get']('features', []) %} + +include: + - elasticsearch.ca + - kafka.sostatus + - kafka.config + - kafka.storage + +so-kafka: + docker_container.running: + - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }} + - hostname: so-kafka + - name: so-kafka + - networks: + - sobridge: + - ipv4_address: {{ DOCKER.containers['so-kafka'].ip }} + - user: kafka + - environment: + KAFKA_HEAP_OPTS: -Xmx2G -Xms1G + KAFKA_OPTS: -javaagent:/opt/jolokia/agents/jolokia-agent-jvm-javaagent.jar=port=8778,host={{ DOCKER.containers['so-kafka'].ip }},policyLocation=file:/opt/jolokia/jolokia.xml + - extra_hosts: + {% for node in KAFKANODES %} + - {{ node }}:{{ KAFKANODES[node].ip }} + {% endfor %} + {% if DOCKER.containers['so-kafka'].extra_hosts %} + {% for XTRAHOST in DOCKER.containers['so-kafka'].extra_hosts %} + - {{ XTRAHOST }} + {% endfor %} + {% endif %} + - port_bindings: + {% for BINDING in DOCKER.containers['so-kafka'].port_bindings %} + - {{ BINDING }} + {% endfor %} + - binds: + - /etc/pki/kafka.p12:/etc/pki/kafka.p12:ro + - /etc/pki/tls/certs/intca.crt:/etc/pki/java/sos/cacerts:ro + - /nsm/kafka/data/:/nsm/kafka/data/:rw + - /opt/so/log/kafka:/opt/kafka/logs/:rw + - /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/server.properties:ro + - /opt/so/conf/kafka/client.properties:/opt/kafka/config/kraft/client.properties + - watch: + {% for sc in ['server', 'client'] %} + - file: kafka_kraft_{{sc}}_properties + {% endfor %} + +delete_so-kafka_so-status.disabled: + file.uncomment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-kafka$ + +{% else %} + +{{sls}}_no_license_detected: + test.fail_without_changes: + - name: {{sls}}_no_license_detected + - comment: + - "Kafka for Guaranteed Message Delivery is a feature supported only for customers with a valid license. + Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com + for more information about purchasing a license to enable this feature." +include: + - kafka.disabled +{% endif %} + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/mysql/map.jinja b/salt/kafka/etc/client.properties.jinja similarity index 67% rename from salt/mysql/map.jinja rename to salt/kafka/etc/client.properties.jinja index dd9a6474e..501b028bf 100644 --- a/salt/mysql/map.jinja +++ b/salt/kafka/etc/client.properties.jinja @@ -3,5 +3,5 @@ https://securityonion.net/license; you may not use this file except in compliance with the Elastic License 2.0. #} -{% import_yaml 'mysql/defaults.yaml' as MYSQLDEFAULTS with context %} -{% set MYSQLMERGED = salt['pillar.get']('mysql', MYSQLDEFAULTS.mysql, merge=True) %} +{% from 'kafka/config.map.jinja' import KAFKACLIENT -%} +{{ KAFKACLIENT | yaml(False) | replace("_x_", ".") }} \ No newline at end of file diff --git a/salt/soctopus/map.jinja b/salt/kafka/etc/server.properties.jinja similarity index 66% rename from salt/soctopus/map.jinja rename to salt/kafka/etc/server.properties.jinja index 07df21dbb..fb0c785cf 100644 --- a/salt/soctopus/map.jinja +++ b/salt/kafka/etc/server.properties.jinja @@ -3,5 +3,5 @@ https://securityonion.net/license; you may not use this file except in compliance with the Elastic License 2.0. #} -{% import_yaml 'soctopus/defaults.yaml' as SOCTOPUSDEFAULTS %} -{% set SOCTOPUSMERGED = salt['pillar.get']('soctopus', SOCTOPUSDEFAULTS.soctopus, merge=True) %} +{% from 'kafka/config.map.jinja' import KAFKACONFIG -%} +{{ KAFKACONFIG | yaml(False) | replace("_x_", ".") }} diff --git a/salt/kafka/files/managed_node_pillar.jinja b/salt/kafka/files/managed_node_pillar.jinja new file mode 100644 index 000000000..aa4f7e502 --- /dev/null +++ b/salt/kafka/files/managed_node_pillar.jinja @@ -0,0 +1,10 @@ +kafka: + nodes: +{% for node, values in COMBINED_KAFKANODES.items() %} + {{ node }}: + ip: {{ values['ip'] }} + nodeid: {{ values['nodeid'] }} +{%- if values['role'] != none %} + role: {{ values['role'] }} +{%- endif %} +{% endfor %} \ No newline at end of file diff --git a/salt/kafka/init.sls b/salt/kafka/init.sls new file mode 100644 index 000000000..0a9b3305f --- /dev/null +++ b/salt/kafka/init.sls @@ -0,0 +1,24 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. +# +# Note: Per the Elastic License 2.0, the second limitation states: +# +# "You may not move, change, disable, or circumvent the license key functionality +# in the software, and you may not remove or obscure any functionality in the +# software that is protected by the license key." + +{% from 'kafka/map.jinja' import KAFKAMERGED %} +{% from 'vars/globals.map.jinja' import GLOBALS %} + +include: +{# Run kafka/nodes.sls before Kafka is enabled, so kafka nodes pillar is setup #} +{% if grains.role in ['so-manager','so-managersearch', 'so-standalone'] %} + - kafka.nodes +{% endif %} +{% if GLOBALS.pipeline == "KAFKA" and KAFKAMERGED.enabled %} + - kafka.enabled +{% else %} + - kafka.disabled +{% endif %} diff --git a/salt/kafka/map.jinja b/salt/kafka/map.jinja new file mode 100644 index 000000000..996e5dedf --- /dev/null +++ b/salt/kafka/map.jinja @@ -0,0 +1,10 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + +{# This is only used to determine if Kafka is enabled / disabled. Configuration is found in kafka/config.map.jinja #} +{# kafka/config.map.jinja depends on there being a kafka nodes pillar being populated #} + +{% import_yaml 'kafka/defaults.yaml' as KAFKADEFAULTS %} +{% set KAFKAMERGED = salt['pillar.get']('kafka', KAFKADEFAULTS.kafka, merge=True) %} diff --git a/salt/kafka/nodes.map.jinja b/salt/kafka/nodes.map.jinja new file mode 100644 index 000000000..e162d3f08 --- /dev/null +++ b/salt/kafka/nodes.map.jinja @@ -0,0 +1,88 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + +{# USED TO GENERATE PILLAR/KAFKA/NODES.SLS. #} +{% import_yaml 'kafka/defaults.yaml' as KAFKADEFAULTS %} +{% from 'vars/globals.map.jinja' import GLOBALS %} + +{% set process_x_roles = KAFKADEFAULTS.kafka.config.broker.process_x_roles %} + +{% set current_kafkanodes = salt.saltutil.runner( + 'mine.get', + tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-receiver', + fun='network.ip_addrs', + tgt_type='compound') %} + +{% set STORED_KAFKANODES = salt['pillar.get']('kafka:nodes', default=None) %} +{% set KAFKA_CONTROLLERS_PILLAR = salt['pillar.get']('kafka:controllers', default=None) %} + +{% set existing_ids = [] %} + +{# Check STORED_KAFKANODES for existing kafka nodes and pull their IDs so they are not reused across the grid #} +{% if STORED_KAFKANODES != none %} +{% for node, values in STORED_KAFKANODES.items() %} +{% if values.get('nodeid') %} +{% do existing_ids.append(values['nodeid']) %} +{% endif %} +{% endfor %} +{% endif %} + +{# Create list of possible node ids #} +{% set all_possible_ids = range(1, 2000)|list %} + +{# Create list of available node ids by looping through all_possible_ids and ensuring it isn't in existing_ids #} +{% set available_ids = [] %} +{% for id in all_possible_ids %} +{% if id not in existing_ids %} +{% do available_ids.append(id) %} +{% endif %} +{% endfor %} + +{# Collect kafka eligible nodes and check if they're already in STORED_KAFKANODES to avoid potentially reassigning a nodeid #} +{% set NEW_KAFKANODES = {} %} +{% for minionid, ip in current_kafkanodes.items() %} +{% set hostname = minionid.split('_')[0] %} +{% if not STORED_KAFKANODES or hostname not in STORED_KAFKANODES %} +{% set new_id = available_ids.pop(0) %} +{% do NEW_KAFKANODES.update({hostname: {'nodeid': new_id, 'ip': ip[0], 'role': process_x_roles }}) %} +{% endif %} +{% endfor %} + +{# Combine STORED_KAFKANODES and NEW_KAFKANODES for writing to the pillar/kafka/nodes.sls #} +{% set COMBINED_KAFKANODES = {} %} +{% for node, details in NEW_KAFKANODES.items() %} +{% do COMBINED_KAFKANODES.update({node: details}) %} +{% endfor %} +{% if STORED_KAFKANODES != none %} +{% for node, details in STORED_KAFKANODES.items() %} +{% do COMBINED_KAFKANODES.update({node: details}) %} +{% endfor %} +{% endif %} + +{# Update the process_x_roles value for any host in the kafka_controllers_pillar configured from SOC UI #} +{% set ns = namespace(has_controller=false) %} +{% if KAFKA_CONTROLLERS_PILLAR != none %} +{% set KAFKA_CONTROLLERS_PILLAR_LIST = KAFKA_CONTROLLERS_PILLAR.split(',') %} +{% for hostname in KAFKA_CONTROLLERS_PILLAR_LIST %} +{% if hostname in COMBINED_KAFKANODES %} +{% do COMBINED_KAFKANODES[hostname].update({'role': 'controller'}) %} +{% set ns.has_controller = true %} +{% endif %} +{% endfor %} +{% for hostname in COMBINED_KAFKANODES %} +{% if hostname not in KAFKA_CONTROLLERS_PILLAR_LIST %} +{% do COMBINED_KAFKANODES[hostname].update({'role': 'broker'}) %} +{% endif %} +{% endfor %} +{# If the kafka_controllers_pillar is NOT empty check that atleast one node contains the controller role. + otherwise default to GLOBALS.manager having broker,controller role #} +{% if not ns.has_controller %} +{% do COMBINED_KAFKANODES[GLOBALS.manager].update({'role': 'broker,controller'}) %} +{% endif %} +{# If kafka_controllers_pillar is empty, default to having grid manager as 'broker,controller' + so there is always atleast 1 controller in the cluster #} +{% else %} +{% do COMBINED_KAFKANODES[GLOBALS.manager].update({'role': 'broker,controller'}) %} +{% endif %} \ No newline at end of file diff --git a/salt/kafka/nodes.sls b/salt/kafka/nodes.sls new file mode 100644 index 000000000..cae2a1d0f --- /dev/null +++ b/salt/kafka/nodes.sls @@ -0,0 +1,18 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'kafka/nodes.map.jinja' import COMBINED_KAFKANODES %} +{% set kafka_cluster_id = salt['pillar.get']('kafka:cluster_id', default=None) %} + +{# Write Kafka pillar, so all grid members have access to nodeid of other kafka nodes and their roles #} +write_kafka_pillar_yaml: + file.managed: + - name: /opt/so/saltstack/local/pillar/kafka/nodes.sls + - mode: 644 + - user: socore + - source: salt://kafka/files/managed_node_pillar.jinja + - template: jinja + - context: + COMBINED_KAFKANODES: {{ COMBINED_KAFKANODES }} \ No newline at end of file diff --git a/salt/mysql/tools/sbin/so-mysql-start b/salt/kafka/reset.sls old mode 100755 new mode 100644 similarity index 79% rename from salt/mysql/tools/sbin/so-mysql-start rename to salt/kafka/reset.sls index e68536809..8789516cd --- a/salt/mysql/tools/sbin/so-mysql-start +++ b/salt/kafka/reset.sls @@ -1,12 +1,9 @@ -#!/bin/bash - # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - - -. /usr/sbin/so-common - -/usr/sbin/so-start mysql $1 +wipe_kafka_data: + file.absent: + - name: /nsm/kafka/data/ + - force: True \ No newline at end of file diff --git a/salt/kafka/soc_kafka.yaml b/salt/kafka/soc_kafka.yaml new file mode 100644 index 000000000..0c9c8a57e --- /dev/null +++ b/salt/kafka/soc_kafka.yaml @@ -0,0 +1,209 @@ +kafka: + enabled: + description: Set to True to enable Kafka. To avoid grid problems, do not enable Kafka until the related configuration is in place. Requires a valid Security Onion license key. + helpLink: kafka.html + cluster_id: + description: The ID of the Kafka cluster. + readonly: True + advanced: True + sensitive: True + helpLink: kafka.html + password: + description: The password to use for the Kafka certificates. + sensitive: True + helpLink: kafka.html + controllers: + description: A comma-separated list of hostnames that will act as Kafka controllers. These hosts will be responsible for managing the Kafka cluster. Note that only manager and receiver nodes are eligible to run Kafka. This configuration needs to be set before enabling Kafka. Failure to do so may result in Kafka topics becoming unavailable requiring manual intervention to restore functionality or reset Kafka, either of which can result in data loss. + forcedType: "string" + helpLink: kafka.html + reset: + description: Disable and reset the Kafka cluster. This will remove all Kafka data including logs that may have not yet been ingested into Elasticsearch and reverts the grid to using REDIS as the global pipeline. This is useful when testing different Kafka configurations such as rearranging Kafka brokers / controllers allowing you to reset the cluster rather than manually fixing any issues arising from attempting to reassign a Kafka broker into a controller. Enter 'YES_RESET_KAFKA' and submit to disable and reset Kafka. Make any configuration changes required and re-enable Kafka when ready. This action CANNOT be reversed. + advanced: True + helpLink: kafka.html + config: + broker: + advertised_x_listeners: + description: Specify the list of listeners (hostname and port) that Kafka brokers provide to clients for communication. + title: advertised.listeners + helpLink: kafka.html + auto_x_create_x_topics_x_enable: + description: Enable the auto creation of topics. + title: auto.create.topics.enable + forcedType: bool + helpLink: kafka.html + default_x_replication_x_factor: + description: The default replication factor for automatically created topics. This value must be less than the amount of brokers in the cluster. Hosts specified in controllers should not be counted towards total broker count. + title: default.replication.factor + forcedType: int + helpLink: kafka.html + inter_x_broker_x_listener_x_name: + description: The name of the listener used for inter-broker communication. + title: inter.broker.listener.name + helpLink: kafka.html + listeners: + description: Set of URIs that is listened on and the listener names in a comma-seperated list. + helpLink: kafka.html + listener_x_security_x_protocol_x_map: + description: Comma-seperated mapping of listener name and security protocols. + title: listener.security.protocol.map + helpLink: kafka.html + log_x_dirs: + description: Where Kafka logs are stored within the Docker container. + title: log.dirs + helpLink: kafka.html + log_x_retention_x_check_x_interval_x_ms: + description: Frequency at which log files are checked if they are qualified for deletion. + title: log.retention.check.interval.ms + helpLink: kafka.html + log_x_retention_x_hours: + description: How long, in hours, a log file is kept. + title: log.retention.hours + forcedType: int + helpLink: kafka.html + log_x_segment_x_bytes: + description: The maximum allowable size for a log file. + title: log.segment.bytes + forcedType: int + helpLink: kafka.html + num_x_io_x_threads: + description: The number of threads used by Kafka. + title: num.io.threads + forcedType: int + helpLink: kafka.html + num_x_network_x_threads: + description: The number of threads used for network communication. + title: num.network.threads + forcedType: int + helpLink: kafka.html + num_x_partitions: + description: The number of log partitions assigned per topic. + title: num.partitions + forcedType: int + helpLink: kafka.html + num_x_recovery_x_threads_x_per_x_data_x_dir: + description: The number of threads used for log recuperation at startup and purging at shutdown. This ammount of threads is used per data directory. + title: num.recovery.threads.per.data.dir + forcedType: int + helpLink: kafka.html + offsets_x_topic_x_replication_x_factor: + description: The offsets topic replication factor. + title: offsets.topic.replication.factor + forcedType: int + helpLink: kafka.html + process_x_roles: + description: The role performed by Kafka brokers. + title: process.roles + readonly: True + helpLink: kafka.html + socket_x_receive_x_buffer_x_bytes: + description: Size, in bytes of the SO_RCVBUF buffer. A value of -1 will use the OS default. + title: socket.receive.buffer.bytes + #forcedType: int - soc needs to allow -1 as an int before we can use this + helpLink: kafka.html + socket_x_request_x_max_x_bytes: + description: The maximum bytes allowed for a request to the socket. + title: socket.request.max.bytes + forcedType: int + helpLink: kafka.html + socket_x_send_x_buffer_x_bytes: + description: Size, in bytes of the SO_SNDBUF buffer. A value of -1 will use the OS default. + title: socket.send.buffer.byte + #forcedType: int - soc needs to allow -1 as an int before we can use this + helpLink: kafka.html + ssl_x_keystore_x_location: + description: The key store file location within the Docker container. + title: ssl.keystore.location + helpLink: kafka.html + ssl_x_keystore_x_password: + description: The key store file password. Invalid for PEM format. + title: ssl.keystore.password + sensitive: True + helpLink: kafka.html + ssl_x_keystore_x_type: + description: The key store file format. + title: ssl.keystore.type + regex: ^(JKS|PKCS12|PEM)$ + helpLink: kafka.html + ssl_x_truststore_x_location: + description: The trust store file location within the Docker container. + title: ssl.truststore.location + helpLink: kafka.html + ssl_x_truststore_x_password: + description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format. + title: ssl.truststore.password + sensitive: True + helpLink: kafka.html + transaction_x_state_x_log_x_min_x_isr: + description: Overrides min.insync.replicas for the transaction topic. When a producer configures acks to "all" (or "-1"), this setting determines the minimum number of replicas required to acknowledge a write as successful. Failure to meet this minimum triggers an exception (either NotEnoughReplicas or NotEnoughReplicasAfterAppend). When used in conjunction, min.insync.replicas and acks enable stronger durability guarantees. For instance, creating a topic with a replication factor of 3, setting min.insync.replicas to 2, and using acks of "all" ensures that the producer raises an exception if a majority of replicas fail to receive a write. + title: transaction.state.log.min.isr + forcedType: int + helpLink: kafka.html + transaction_x_state_x_log_x_replication_x_factor: + description: Set the replication factor higher for the transaction topic to ensure availability. Internal topic creation will not proceed until the cluster size satisfies this replication factor prerequisite. + title: transaction.state.log.replication.factor + forcedType: int + helpLink: kafka.html + client: + security_x_protocol: + description: 'Broker communication protocol. Options are: SASL_SSL, PLAINTEXT, SSL, SASL_PLAINTEXT' + title: security.protocol + regex: ^(SASL_SSL|PLAINTEXT|SSL|SASL_PLAINTEXT) + helpLink: kafka.html + ssl_x_keystore_x_location: + description: The key store file location within the Docker container. + title: ssl.keystore.location + helpLink: kafka.html + ssl_x_keystore_x_password: + description: The key store file password. Invalid for PEM format. + title: ssl.keystore.password + sensitive: True + helpLink: kafka.html + ssl_x_keystore_x_type: + description: The key store file format. + title: ssl.keystore.type + regex: ^(JKS|PKCS12|PEM)$ + helpLink: kafka.html + ssl_x_truststore_x_location: + description: The trust store file location within the Docker container. + title: ssl.truststore.location + helpLink: kafka.html + ssl_x_truststore_x_password: + description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format. + title: ssl.truststore.password + sensitive: True + helpLink: kafka.html + controller: + controller_x_listener_x_names: + description: Set listeners used by the controller in a comma-seperated list. + title: controller.listener.names + helpLink: kafka.html + listeners: + description: Set of URIs that is listened on and the listener names in a comma-seperated list. + helpLink: kafka.html + listener_x_security_x_protocol_x_map: + description: Comma-seperated mapping of listener name and security protocols. + title: listener.security.protocol.map + helpLink: kafka.html + log_x_dirs: + description: Where Kafka logs are stored within the Docker container. + title: log.dirs + helpLink: kafka.html + log_x_retention_x_check_x_interval_x_ms: + description: Frequency at which log files are checked if they are qualified for deletion. + title: log.retention.check.interval.ms + helpLink: kafka.html + log_x_retention_x_hours: + description: How long, in hours, a log file is kept. + title: log.retention.hours + forcedType: int + helpLink: kafka.html + log_x_segment_x_bytes: + description: The maximum allowable size for a log file. + title: log.segment.bytes + forcedType: int + helpLink: kafka.html + process_x_roles: + description: The role performed by controller node. + title: process.roles + readonly: True + helpLink: kafka.html \ No newline at end of file diff --git a/salt/mysql/sostatus.sls b/salt/kafka/sostatus.sls similarity index 82% rename from salt/mysql/sostatus.sls rename to salt/kafka/sostatus.sls index 2f5dbba06..37c868a46 100644 --- a/salt/mysql/sostatus.sls +++ b/salt/kafka/sostatus.sls @@ -6,11 +6,11 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} -append_so-mysql_so-status.conf: +append_so-kafka_so-status.conf: file.append: - name: /opt/so/conf/so-status/so-status.conf - - text: so-mysql - - unless: grep -q so-mysql /opt/so/conf/so-status/so-status.conf + - text: so-kafka + - unless: grep -q so-kafka /opt/so/conf/so-status/so-status.conf {% else %} @@ -18,4 +18,4 @@ append_so-mysql_so-status.conf: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} +{% endif %} \ No newline at end of file diff --git a/salt/kafka/storage.sls b/salt/kafka/storage.sls new file mode 100644 index 000000000..efc36acf6 --- /dev/null +++ b/salt/kafka/storage.sls @@ -0,0 +1,30 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% set kafka_cluster_id = salt['pillar.get']('kafka:cluster_id') %} + +{# Initialize kafka storage if it doesn't already exist. Just looking for meta.properties in /nsm/kafka/data #} +{% if not salt['file.file_exists']('/nsm/kafka/data/meta.properties') %} +kafka_storage_init: + cmd.run: + - name: | + docker run -v /nsm/kafka/data:/nsm/kafka/data -v /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/newserver.properties --name so-kafkainit --user root --entrypoint /opt/kafka/bin/kafka-storage.sh {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }} format -t {{ kafka_cluster_id }} -c /opt/kafka/config/kraft/newserver.properties +kafka_rm_kafkainit: + cmd.run: + - name: | + docker rm so-kafkainit +{% endif %} + + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/kafka/tools/sbin/so-kafka-cli b/salt/kafka/tools/sbin/so-kafka-cli new file mode 100644 index 000000000..41993f67f --- /dev/null +++ b/salt/kafka/tools/sbin/so-kafka-cli @@ -0,0 +1,47 @@ +#! /bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +if [ -z "$NOROOT" ]; then + # Check for prerequisites + if [ "$(id -u)" -ne 0 ]; then + echo "This script must be run using sudo!" + exit 1 + fi +fi + +function usage() { + echo -e "\nUsage: $0